Robert Elder commited on
Commit
4870db0
·
1 Parent(s): cdf4931

disabling DSSTOX scraping since it is slow and likely to fail

Browse files
Files changed (2) hide show
  1. CHRIS.py +18 -18
  2. ChemID.py +205 -204
CHRIS.py CHANGED
@@ -12,24 +12,24 @@ app.debug = True
12
  #)
13
 
14
  ### DEBUGGING
15
- #if 0:
16
- # from pprint import pformat
17
- # from flask import g, request
18
- #
19
- # @app.after_request
20
- # def dump_chem_trace(response):
21
- # trace = getattr(g, "chem_trace", None)
22
- # if trace:
23
- # bases = sorted(set(getattr(g, "chem_bases", [])))
24
- # payload = {
25
- # "path": request.path,
26
- # "query": request.values.to_dict(flat=False),
27
- # "base_urls": bases,
28
- # "remote_trace": [x for x in trace if x.get("kind") == "remote"],
29
- # "app_trace": [x for x in trace if x.get("kind") == "app"],
30
- # }
31
- # app.logger.warning("CHEMTRACE\n%s", pformat(payload, sort_dicts=False, width=120))
32
- # return response
33
 
34
  #from color_module import blueprint
35
  #app.register_blueprint(blueprint)
 
12
  #)
13
 
14
  ### DEBUGGING
15
+ if 0:
16
+ from pprint import pformat
17
+ from flask import g, request
18
+
19
+ @app.after_request
20
+ def dump_chem_trace(response):
21
+ trace = getattr(g, "chem_trace", None)
22
+ if trace:
23
+ bases = sorted(set(getattr(g, "chem_bases", [])))
24
+ payload = {
25
+ "path": request.path,
26
+ "query": request.values.to_dict(flat=False),
27
+ "base_urls": bases,
28
+ "remote_trace": [x for x in trace if x.get("kind") == "remote"],
29
+ "app_trace": [x for x in trace if x.get("kind") == "app"],
30
+ }
31
+ app.logger.warning("CHEMTRACE\n%s", pformat(payload, sort_dicts=False, width=120))
32
+ return response
33
 
34
  #from color_module import blueprint
35
  #app.register_blueprint(blueprint)
ChemID.py CHANGED
@@ -52,177 +52,177 @@ _force_disable_cirpy = os.environ.get(
52
 
53
  try_cirpy = False if _force_disable_cirpy else _check_cirpy_access()
54
 
55
- #if 0:
56
- # ## DEBUGGING
57
- # import time
58
- # import functools
59
- # import inspect
60
- # import logging
61
- # import os
62
- # import urllib.request
63
- # import urllib.parse
64
- #
65
- # try:
66
- # from flask import has_request_context, g
67
- # except Exception:
68
- # def has_request_context():
69
- # return False
70
- # g = None
71
- #
72
- # TRACE_THRESHOLD = float(os.environ.get("CHEM_TRACE_THRESHOLD", "0.0"))
73
- #
74
- # chemtrace_logger = logging.getLogger("chemtrace")
75
- # if not chemtrace_logger.handlers:
76
- # chemtrace_logger.setLevel(logging.INFO)
77
- #
78
- # def _short(x, n=200):
79
- # s = repr(x)
80
- # return s if len(s) <= n else s[:n-3] + "..."
81
- #
82
- # def _find_app_caller():
83
- # for frame in inspect.stack()[2:]:
84
- # fn = frame.filename.replace("\\", "/")
85
- # if (
86
- # "site-packages" not in fn
87
- # and "/lib/python" not in fn
88
- # and not fn.endswith("/logging/__init__.py")
89
- # ):
90
- # return {
91
- # "file": os.path.basename(frame.filename),
92
- # "func": frame.function,
93
- # "line": frame.lineno,
94
- # }
95
- # return None
96
- #
97
- # def _url_info(url):
98
- # if not url:
99
- # return None
100
- # try:
101
- # parts = urllib.parse.urlsplit(str(url))
102
- # base = f"{parts.scheme}://{parts.netloc}" if parts.scheme and parts.netloc else None
103
- # return {
104
- # "url": str(url),
105
- # "base": base,
106
- # "host": parts.netloc or None,
107
- # "path": parts.path or "",
108
- # }
109
- # except Exception:
110
- # return {"url": _short(url), "base": None, "host": None, "path": None}
111
- #
112
- # def _append_trace_event(event):
113
- # if has_request_context():
114
- # if getattr(g, "chem_trace", None) is None:
115
- # g.chem_trace = []
116
- # g.chem_trace.append(event)
117
- #
118
- # info = event.get("target")
119
- # base = info.get("base") if isinstance(info, dict) else None
120
- # if base:
121
- # if getattr(g, "chem_bases", None) is None:
122
- # g.chem_bases = []
123
- # g.chem_bases.append(base)
124
- # else:
125
- # chemtrace_logger.info("%r", event)
126
- #
127
- # def trace_call(fn):
128
- # @functools.wraps(fn)
129
- # def wrapper(*args, **kwargs):
130
- # t0 = time.perf_counter()
131
- # ok = False
132
- # err = None
133
- # try:
134
- # result = fn(*args, **kwargs)
135
- # ok = True
136
- # return result
137
- # except Exception as e:
138
- # err = f"{type(e).__name__}: {e}"
139
- # raise
140
- # finally:
141
- # dt = time.perf_counter() - t0
142
- # if dt >= TRACE_THRESHOLD:
143
- # _append_trace_event({
144
- # "kind": "app",
145
- # "fn": fn.__name__,
146
- # "secs": round(dt, 4),
147
- # "arg0": _short(args[0]) if args else None,
148
- # "ok": ok,
149
- # "err": err,
150
- # })
151
- # return wrapper
152
- #
153
- # def trace_external(label, fn, formatter=None):
154
- # @functools.wraps(fn)
155
- # def wrapper(*args, **kwargs):
156
- # t0 = time.perf_counter()
157
- # ok = False
158
- # err = None
159
- # caller = _find_app_caller()
160
- # target = None
161
- # if formatter is not None:
162
- # try:
163
- # target = formatter(args, kwargs)
164
- # except Exception:
165
- # target = None
166
- # try:
167
- # result = fn(*args, **kwargs)
168
- # ok = True
169
- # return result
170
- # except Exception as e:
171
- # err = f"{type(e).__name__}: {e}"
172
- # raise
173
- # finally:
174
- # dt = time.perf_counter() - t0
175
- # if dt >= TRACE_THRESHOLD:
176
- # _append_trace_event({
177
- # "kind": "remote",
178
- # "fn": label,
179
- # "caller": caller,
180
- # "target": target,
181
- # "secs": round(dt, 4),
182
- # "ok": ok,
183
- # "err": err,
184
- # })
185
- # return wrapper
186
- #
187
- # def _fmt_urlopen(args, kwargs):
188
- # url = args[0] if len(args) > 0 else kwargs.get("url")
189
- # info = _url_info(url)
190
- # if info is not None:
191
- # info["timeout"] = kwargs.get("timeout")
192
- # return info
193
- #
194
- # def _fmt_requests(args, kwargs):
195
- # method = args[1] if len(args) > 1 else kwargs.get("method")
196
- # url = args[2] if len(args) > 2 else kwargs.get("url")
197
- # info = _url_info(url)
198
- # if info is not None:
199
- # info["method"] = method
200
- # info["timeout"] = kwargs.get("timeout")
201
- # return info
202
- #
203
- # def _fmt_selenium_get(args, kwargs):
204
- # url = args[1] if len(args) > 1 else kwargs.get("url")
205
- # return _url_info(url)
206
- #
207
- # def _fmt_pcp(args, kwargs):
208
- # query = args[0] if len(args) > 0 else kwargs.get("identifier")
209
- # namespace = args[1] if len(args) > 1 else kwargs.get("namespace")
210
- # info = _url_info(getattr(pcp, "API_BASE", None)) or {}
211
- # info.update({
212
- # "query": _short(query),
213
- # "namespace": namespace,
214
- # })
215
- # return info
216
- #
217
- # def _fmt_cirpy(args, kwargs):
218
- # query = args[0] if len(args) > 0 else None
219
- # resolver = args[1] if len(args) > 1 else kwargs.get("resolver")
220
- # info = _url_info(getattr(cirpy, "API_BASE", None)) or {}
221
- # info.update({
222
- # "query": _short(query),
223
- # "resolver": resolver,
224
- # })
225
- # return info
226
 
227
  ORGANIC_ATOM_SET = {5, 6, 7, 8, 9, 15, 16, 17, 35, 53}
228
  METAL_ATOM_SET = set([3,4,11,12,13] + list(range(19,31+1)) + list(range(37,50+1)) + list(range(55,84+1)) + list(range(87,114+1)) + [116])
@@ -236,7 +236,7 @@ SALT_SET = {line.strip() for line in lines}
236
  ERROR_CODES = {0:None, 1:'Structure could not be determined from the identifier', 2:'Invalid SMILES code', 3:'Invalid CAS number', 4:'Invalid identifier type selected', 5:'Invalid InChI'}
237
 
238
  ## not sure if this will be possible on pythonanywhere; use this flag to disable related code blocks
239
- try_dsstox = True
240
  if try_dsstox:
241
  import selenium
242
  import selenium.webdriver
@@ -1407,35 +1407,36 @@ def getLogP(cas,mol):
1407
  return LogP, LogP_origin
1408
 
1409
  #### DEBUGGING
1410
- #if 0:
1411
- # ResolveChemical = trace_call(ResolveChemical)
1412
- # smiles2name = trace_call(smiles2name)
1413
- # cas2smiles = trace_call(cas2smiles)
1414
- # cas2name = trace_call(cas2name)
1415
- # name2iupac = trace_call(name2iupac)
1416
- # name2cas = trace_call(name2cas)
1417
- # name2smiles = trace_call(name2smiles)
1418
- # string2density = trace_call(string2density)
1419
- # string2mp = trace_call(string2mp)
1420
- # mol2mp = trace_call(mol2mp)
1421
- # if hasattr(pcp, "urlopen"):
1422
- # pcp.urlopen = trace_external("pubchempy.urlopen", pcp.urlopen, _fmt_urlopen)
1423
- #
1424
- # if hasattr(cirpy, "urlopen"):
1425
- # cirpy.urlopen = trace_external("cirpy.urlopen", cirpy.urlopen, _fmt_urlopen)
1426
- #
1427
- # pcp.get_compounds = trace_external("pcp.get_compounds", pcp.get_compounds, _fmt_pcp)
1428
- # pcp.get_substances = trace_external("pcp.get_substances", pcp.get_substances, _fmt_pcp)
1429
- # cirpy.resolve = trace_external("cirpy.resolve", cirpy.resolve, _fmt_cirpy)
1430
- # urllib.request.urlopen = trace_external("urllib.request.urlopen", urllib.request.urlopen, _fmt_urlopen)
1431
- # requests.sessions.Session.request = trace_external(
1432
- # "requests.Session.request",
1433
- # requests.sessions.Session.request,
1434
- # _fmt_requests,
1435
- # )
1436
- #
1437
- # if try_dsstox:
1438
- # selenium.webdriver.Chrome.get = trace_external(
1439
- # "selenium.webdriver.Chrome.get",
1440
- # selenium.webdriver.Chrome.get,
1441
- # _fmt_selenium_get,
 
 
52
 
53
  try_cirpy = False if _force_disable_cirpy else _check_cirpy_access()
54
 
55
+ if 0:
56
+ ## DEBUGGING
57
+ import time
58
+ import functools
59
+ import inspect
60
+ import logging
61
+ import os
62
+ import urllib.request
63
+ import urllib.parse
64
+
65
+ try:
66
+ from flask import has_request_context, g
67
+ except Exception:
68
+ def has_request_context():
69
+ return False
70
+ g = None
71
+
72
+ TRACE_THRESHOLD = float(os.environ.get("CHEM_TRACE_THRESHOLD", "0.0"))
73
+
74
+ chemtrace_logger = logging.getLogger("chemtrace")
75
+ if not chemtrace_logger.handlers:
76
+ chemtrace_logger.setLevel(logging.INFO)
77
+
78
+ def _short(x, n=200):
79
+ s = repr(x)
80
+ return s if len(s) <= n else s[:n-3] + "..."
81
+
82
+ def _find_app_caller():
83
+ for frame in inspect.stack()[2:]:
84
+ fn = frame.filename.replace("\\", "/")
85
+ if (
86
+ "site-packages" not in fn
87
+ and "/lib/python" not in fn
88
+ and not fn.endswith("/logging/__init__.py")
89
+ ):
90
+ return {
91
+ "file": os.path.basename(frame.filename),
92
+ "func": frame.function,
93
+ "line": frame.lineno,
94
+ }
95
+ return None
96
+
97
+ def _url_info(url):
98
+ if not url:
99
+ return None
100
+ try:
101
+ parts = urllib.parse.urlsplit(str(url))
102
+ base = f"{parts.scheme}://{parts.netloc}" if parts.scheme and parts.netloc else None
103
+ return {
104
+ "url": str(url),
105
+ "base": base,
106
+ "host": parts.netloc or None,
107
+ "path": parts.path or "",
108
+ }
109
+ except Exception:
110
+ return {"url": _short(url), "base": None, "host": None, "path": None}
111
+
112
+ def _append_trace_event(event):
113
+ if has_request_context():
114
+ if getattr(g, "chem_trace", None) is None:
115
+ g.chem_trace = []
116
+ g.chem_trace.append(event)
117
+
118
+ info = event.get("target")
119
+ base = info.get("base") if isinstance(info, dict) else None
120
+ if base:
121
+ if getattr(g, "chem_bases", None) is None:
122
+ g.chem_bases = []
123
+ g.chem_bases.append(base)
124
+ else:
125
+ chemtrace_logger.info("%r", event)
126
+
127
+ def trace_call(fn):
128
+ @functools.wraps(fn)
129
+ def wrapper(*args, **kwargs):
130
+ t0 = time.perf_counter()
131
+ ok = False
132
+ err = None
133
+ try:
134
+ result = fn(*args, **kwargs)
135
+ ok = True
136
+ return result
137
+ except Exception as e:
138
+ err = f"{type(e).__name__}: {e}"
139
+ raise
140
+ finally:
141
+ dt = time.perf_counter() - t0
142
+ if dt >= TRACE_THRESHOLD:
143
+ _append_trace_event({
144
+ "kind": "app",
145
+ "fn": fn.__name__,
146
+ "secs": round(dt, 4),
147
+ "arg0": _short(args[0]) if args else None,
148
+ "ok": ok,
149
+ "err": err,
150
+ })
151
+ return wrapper
152
+
153
+ def trace_external(label, fn, formatter=None):
154
+ @functools.wraps(fn)
155
+ def wrapper(*args, **kwargs):
156
+ t0 = time.perf_counter()
157
+ ok = False
158
+ err = None
159
+ caller = _find_app_caller()
160
+ target = None
161
+ if formatter is not None:
162
+ try:
163
+ target = formatter(args, kwargs)
164
+ except Exception:
165
+ target = None
166
+ try:
167
+ result = fn(*args, **kwargs)
168
+ ok = True
169
+ return result
170
+ except Exception as e:
171
+ err = f"{type(e).__name__}: {e}"
172
+ raise
173
+ finally:
174
+ dt = time.perf_counter() - t0
175
+ if dt >= TRACE_THRESHOLD:
176
+ _append_trace_event({
177
+ "kind": "remote",
178
+ "fn": label,
179
+ "caller": caller,
180
+ "target": target,
181
+ "secs": round(dt, 4),
182
+ "ok": ok,
183
+ "err": err,
184
+ })
185
+ return wrapper
186
+
187
+ def _fmt_urlopen(args, kwargs):
188
+ url = args[0] if len(args) > 0 else kwargs.get("url")
189
+ info = _url_info(url)
190
+ if info is not None:
191
+ info["timeout"] = kwargs.get("timeout")
192
+ return info
193
+
194
+ def _fmt_requests(args, kwargs):
195
+ method = args[1] if len(args) > 1 else kwargs.get("method")
196
+ url = args[2] if len(args) > 2 else kwargs.get("url")
197
+ info = _url_info(url)
198
+ if info is not None:
199
+ info["method"] = method
200
+ info["timeout"] = kwargs.get("timeout")
201
+ return info
202
+
203
+ def _fmt_selenium_get(args, kwargs):
204
+ url = args[1] if len(args) > 1 else kwargs.get("url")
205
+ return _url_info(url)
206
+
207
+ def _fmt_pcp(args, kwargs):
208
+ query = args[0] if len(args) > 0 else kwargs.get("identifier")
209
+ namespace = args[1] if len(args) > 1 else kwargs.get("namespace")
210
+ info = _url_info(getattr(pcp, "API_BASE", None)) or {}
211
+ info.update({
212
+ "query": _short(query),
213
+ "namespace": namespace,
214
+ })
215
+ return info
216
+
217
+ def _fmt_cirpy(args, kwargs):
218
+ query = args[0] if len(args) > 0 else None
219
+ resolver = args[1] if len(args) > 1 else kwargs.get("resolver")
220
+ info = _url_info(getattr(cirpy, "API_BASE", None)) or {}
221
+ info.update({
222
+ "query": _short(query),
223
+ "resolver": resolver,
224
+ })
225
+ return info
226
 
227
  ORGANIC_ATOM_SET = {5, 6, 7, 8, 9, 15, 16, 17, 35, 53}
228
  METAL_ATOM_SET = set([3,4,11,12,13] + list(range(19,31+1)) + list(range(37,50+1)) + list(range(55,84+1)) + list(range(87,114+1)) + [116])
 
236
  ERROR_CODES = {0:None, 1:'Structure could not be determined from the identifier', 2:'Invalid SMILES code', 3:'Invalid CAS number', 4:'Invalid identifier type selected', 5:'Invalid InChI'}
237
 
238
  ## not sure if this will be possible on pythonanywhere; use this flag to disable related code blocks
239
+ try_dsstox = False
240
  if try_dsstox:
241
  import selenium
242
  import selenium.webdriver
 
1407
  return LogP, LogP_origin
1408
 
1409
  #### DEBUGGING
1410
+ if 0:
1411
+ ResolveChemical = trace_call(ResolveChemical)
1412
+ smiles2name = trace_call(smiles2name)
1413
+ cas2smiles = trace_call(cas2smiles)
1414
+ cas2name = trace_call(cas2name)
1415
+ name2iupac = trace_call(name2iupac)
1416
+ name2cas = trace_call(name2cas)
1417
+ name2smiles = trace_call(name2smiles)
1418
+ string2density = trace_call(string2density)
1419
+ string2mp = trace_call(string2mp)
1420
+ mol2mp = trace_call(mol2mp)
1421
+ if hasattr(pcp, "urlopen"):
1422
+ pcp.urlopen = trace_external("pubchempy.urlopen", pcp.urlopen, _fmt_urlopen)
1423
+
1424
+ if hasattr(cirpy, "urlopen"):
1425
+ cirpy.urlopen = trace_external("cirpy.urlopen", cirpy.urlopen, _fmt_urlopen)
1426
+
1427
+ pcp.get_compounds = trace_external("pcp.get_compounds", pcp.get_compounds, _fmt_pcp)
1428
+ pcp.get_substances = trace_external("pcp.get_substances", pcp.get_substances, _fmt_pcp)
1429
+ cirpy.resolve = trace_external("cirpy.resolve", cirpy.resolve, _fmt_cirpy)
1430
+ urllib.request.urlopen = trace_external("urllib.request.urlopen", urllib.request.urlopen, _fmt_urlopen)
1431
+ requests.sessions.Session.request = trace_external(
1432
+ "requests.Session.request",
1433
+ requests.sessions.Session.request,
1434
+ _fmt_requests,
1435
+ )
1436
+
1437
+ if try_dsstox:
1438
+ selenium.webdriver.Chrome.get = trace_external(
1439
+ "selenium.webdriver.Chrome.get",
1440
+ selenium.webdriver.Chrome.get,
1441
+ _fmt_selenium_get,
1442
+ )