diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..a49ea1273b1e6af3b5340dafa43f9a1bf3e4d30a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +scripts/stride/stride filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..6c31a026bca7079eaab5f521de44d66577a86f04 --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +.git/ +.idea/ +.vscode/ +__pycache__/ +*.py[cod] +*.o +*.so +*.dylib +*.DS_Store +*.ipynb_checkpoints/ + +# Runtime and test outputs +output/ +results/ +conv_logs/ +*.png +*.npy +*_renum.pdb +*image_gen.pml +*.crh + +# Local environments +venv/ +chswEnv/ + +# Local datasets and experiment outputs +pdbs/ +scripts/development/experiments/cath_annotated_alphafold/assignment.table +model_v3_predictions* diff --git a/.ms_upload_cache b/.ms_upload_cache new file mode 100644 index 0000000000000000000000000000000000000000..e4133c3fb935bcbe25f7da3faa74c2dd4b906f5d --- /dev/null +++ b/.ms_upload_cache @@ -0,0 +1 @@ +{"version": 3, "repo_id": "OneScience/Chainsaw", "files": {"README.md|1787816209.0|5989": {"hash": "56808cd07e168b6203f9aa773d0e0684b79be83d560bfb88a5b1c5243360b4b1", "size": 5989, "status": "c"}, ".gitignore|1787814402.0|375": {"hash": "e932d06e9b9b8d3321a9f8eec7c56c431b542201bf999856acb3fb30fa21d7b1", "size": 375, "status": "c"}, "conf/model_v2/feature_config.json|1785482345.0|312": {"hash": "1a0bc59e8de8f07a56485757cc196b33f7aabb3bb170d42998f49796003658ff", "size": 312, "status": "c"}, "conf/config.json|1787814155.0|384": {"hash": "6b2fbe7080161d45ef1f3a8fa26f46359386e12ea2d8f6b28ac6a03250fce36a", "size": 384, "status": "c"}, "conf/model_v1/feature_config.json|1785482345.0|294": {"hash": "787953dbcdbe4c96c1b5bb0d8011e322f6cfab44b5ee72a156e94c625d59e545", "size": 294, "status": "c"}, "conf/model_v2/config.json|1785482345.0|1718": {"hash": "182cc72b36b5fd7321cae3942174938ac18aa304df853341506f095bb40b5509", "size": 1718, "status": "c"}, "conf/model_v1/config.json|1785482345.0|1582": {"hash": "23d97d37f86ea0a812f11eecd9dc22fd27af54ae2af81981438faf4fe273e576", "size": 1582, "status": "c"}, "conf/LICENSE|1785482344.0|1067": {"hash": "f4d5206446ac533b01d5a51875564b2237a7d81299d26afd7a9524622b00d268", "size": 1067, "status": "c"}, "conf/model_v3/config.json|1785482345.0|1978": {"hash": "fdf3ea1d9e6c2c72ed63cf04952a6845258cc725ec8f6f529aa577bb5d550715", "size": 1978, "status": "c"}, "conf/model_v3/feature_config.json|1785482345.0|304": {"hash": "5b0ab82dbdfd012b403d27ef9f1ec959bcadc7ff12427a0d9e8a9d2da4c16522", "size": 304, "status": "c"}, "model/__init__.py|1785482345.0|0": {"hash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size": 0, "status": "c"}, "conf/pyproject.toml|1785482344.0|29": {"hash": "dd3b03f9d41e0effb906878d84dcb095a50ab162e0b707d3db9bfb9e61aa914e", "size": 29, "status": "c"}, "conf/requirements_to_freeze.dev.txt|1785482344.0|6": {"hash": "2b01214d32c382dbc73bf6b493bedb7926324af27bad3ee888f66774b9b114c8", "size": 6, "status": "c"}, "conf/requirements.txt|1787814152.0|191": {"hash": "c5b28226aefbb9e5280dc221463c7b63738322c0ea0d9795262ddc0a1d42b01a", "size": 191, "status": "c"}, "configuration.json|1787815553.0|328": {"hash": "5e26bc304c4f9c6bb74d0337fc9a6e37f940caa96c111e8f4e7d150a31edb2b8", "size": 328, "status": "c"}, "conf/requirements_to_freeze.txt|1785482344.0|63": {"hash": "18962dbd2998744de9737aa6aaf7ce87585d035e06c49422a0f0bd0e894f814c", "size": 63, "status": "c"}, "model/__pycache__/domain_chop.cpython-311.pyc|1787816725.0|22343": {"hash": "1ae6e101441745b8c96a1f8a2d5050289e7e966228e3f5253d98f519cf8756a4", "size": 22343, "status": "c"}, "model/__pycache__/constants.cpython-311.pyc|1787816724.0|1159": {"hash": "b07825ba20304672a3e2e2b3878f9087d253b1f0de01b592841f02f67564c05b", "size": 1159, "status": "c"}, "model/__pycache__/__init__.cpython-311.pyc|1787816724.0|164": {"hash": "fabc8e36d0a6869e2538c59765c36eaed2007c223280fbcfa6e9a57f359490ec", "size": 164, "status": "c"}, "model/__pycache__/errors.cpython-311.pyc|1787816726.0|894": {"hash": "f080a1cc2ae92960a27ccca53417eb09b1540621208de6b5674db2755a542a8b", "size": 894, "status": "c"}, "model/constants.py|1787813883.0|542": {"hash": "87c6b8c8cb999d658216c21a7d473b23f7c25e5c245d91e154df81f3fae02b20", "size": 542, "status": "c"}, "model/__pycache__/factories.cpython-311.pyc|1787816725.0|4196": {"hash": "cd4a768e04a1d448957a3eb66d22ce41a9f9d21b79800480ef36e9082113c4d3", "size": 4196, "status": "c"}, "model/__pycache__/featurisers.cpython-311.pyc|1787816724.0|8857": {"hash": "4e7f4101d5ed373cb5b3ce3bcf03e51aa024ddf33cc5a87b6a6460781a985835", "size": 8857, "status": "c"}, "model/__pycache__/prediction_result_file.cpython-311.pyc|1787816726.0|9258": {"hash": "26a27e14427cc6006a9b4b1133cba1fda66a602414cb40c11e6d1e7ead6c62ec", "size": 9258, "status": "c"}, "model/domain_assignment/__pycache__/assigners.cpython-311.pyc|1787816725.0|5014": {"hash": "96973c7770ab6dc832d0d6d1304ec2bc2538f38a5f35003574e5ff6e3e1da434", "size": 5014, "status": "c"}, "model/domain_assignment/__pycache__/sparse_lowrank.cpython-311.pyc|1787816725.0|5486": {"hash": "ade3050279be92b9a6ee89ad791d2bcaf348d27cdb13053792062a25fcf5b293", "size": 5486, "status": "c"}, "model/domain_assignment/__pycache__/util.cpython-311.pyc|1787816725.0|8283": {"hash": "bcc89d70b56223006212c5573caa7ca556b08661a02c69d39c9bf857f560365f", "size": 8283, "status": "c"}, "model/domain_assignment/util.py|1785482345.0|4708": {"hash": "007718b5575d2078a5fc14c126ca56d21ace273756c5ecbc96d219d458fc05ff", "size": 4708, "status": "c"}, "model/domain_assignment/assigners.py|1787813888.0|2601": {"hash": "9ae0d882dbebc9e3136b6c3764780432455554e2456a90f6d381d52808a6feff", "size": 2601, "status": "c"}, "model/domain_assignment/sparse_lowrank.py|1785482345.0|4740": {"hash": "cebf40615a983cb8adf69bca8d26f133b13c3d5bbb798267f7d2179aba9235a0", "size": 4740, "status": "c"}, "model/featurisers.py|1787813887.0|5638": {"hash": "bfa24e1bd67fc571fbea274ed667e9f99b8c5a4090298fdc823c0865c8796fc1", "size": 5638, "status": "c"}, "model/domain_chop.py|1787813884.0|13864": {"hash": "a1d7c93aeb381c6bbf1aac67b8156ab6aac0bb3181640993afd45973275443d9", "size": 13864, "status": "c"}, "model/factories.py|1787813886.0|2253": {"hash": "89fb7816fc24be8986e4767e9e67b3b49225adbe63147b2af523acd9b814191d", "size": 2253, "status": "c"}, "model/errors.py|1785482345.0|211": {"hash": "a7975fbfc65fc42493728d5505da08ea5178eaf736c696ea87c1102948fca4f5", "size": 211, "status": "c"}, "model/loggers.py|1785482345.0|4644": {"hash": "e7d3eb982b79b182136fa85941664a219eb4b8999c8e42fb916aa47e1437694f", "size": 4644, "status": "c"}, "model/models/__pycache__/results.cpython-311.pyc|1787816726.0|1501": {"hash": "71445e02af3d65638406e62f9eac236670fe667cc5147e36a10a7010fe815fc3", "size": 1501, "status": "c"}, "model/utils/__init__.py|1785482345.0|0": {"hash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size": 0, "status": "c"}, "model/models/results.py|1787814533.0|608": {"hash": "49509469b2f29a7ef0406e979f96530927c96ab75bfd156ebde8a2ece619eefd", "size": 608, "status": "c"}, "model/models/__pycache__/rosetta.cpython-311.pyc|1787816725.0|4770": {"hash": "01681890fc010794b87b539df5c1f88d8a0790549e97a0f5275ce3c535d11451", "size": 4770, "status": "c"}, "model/prediction_result_file.py|1785482345.0|5365": {"hash": "aa35107f7daea5a9b4918aa18539bd481aed156fa3f9bc527e462fb5f6ff8766", "size": 5365, "status": "c"}, "model/models/rosetta.py|1785482345.0|2335": {"hash": "b6a378368d45481361dd8f0f5e0afcfd45b487cba4927c267a06382d2718029f", "size": 2335, "status": "c"}, "model/utils/__pycache__/__init__.cpython-311.pyc|1787816725.0|170": {"hash": "8e7d302e9f279abc1cb354eeea6af673ae5ab313a1abb9309b6b1ceff21cabfb", "size": 170, "status": "c"}, "model/utils/__pycache__/common.cpython-311.pyc|1787816725.0|2926": {"hash": "b0e4cb25ee74d750e0661e37342540ac260839996b04ca2126e6c1fcebfcb357", "size": 2926, "status": "c"}, "model/utils/__pycache__/cif2pdb.cpython-311.pyc|1787816725.0|6773": {"hash": "674f8ca7af4c0f8180bd1dc6abe8cfb278b9ea185c65f828cf6bdb8fa61a04fa", "size": 6773, "status": "c"}, "model/utils/__pycache__/secondary_structure.cpython-311.pyc|1787816725.0|7245": {"hash": "5996a7911198ae8d5b3d30063950df84284ea52483389158ab5df53126a8903f", "size": 7245, "status": "c"}, "model/utils/__pycache__/pymol_3d_visuals.cpython-311.pyc|1787816726.0|2913": {"hash": "d47671d6b97ed9c3903b7f331e1140fb66aa7d81984bf03b158f35ceebe438bf", "size": 2913, "status": "c"}, "model/utils/cif2pdb.py|1785482345.0|4247": {"hash": "4c3848a69a1968a918ce8700950dc4173533f74945ebebbe6153e1423d82e130", "size": 4247, "status": "c"}, "model/utils/ndo_score.py|1785482345.0|4073": {"hash": "c3e2e8a6882dd3c8dc612ffeaf3355c499821c3e9926b66a81747830701e4215", "size": 4073, "status": "c"}, "model/utils/common.py|1785482345.0|1260": {"hash": "d23404e29a3ba78f9b30996534391353627e6e2be26ca86db944999851aa2522", "size": 1260, "status": "c"}, "scripts/__init__.py|1785482345.0|0": {"hash": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", "size": 0, "status": "c"}, "model/utils/domain_boundary_distance_score.py|1785482345.0|3575": {"hash": "ce6fb5667f735080e8696de71b0b8618becb720adfca798ea8293bbb5c3d4a84", "size": 3575, "status": "c"}, "model/utils/pymol_3d_visuals.py|1785482345.0|1972": {"hash": "88bc47afff4a41032e714a5a2cb5bf26be162ff3c868fd5263034a26c367f011", "size": 1972, "status": "c"}, "model/utils/secondary_structure.py|1787814846.0|3918": {"hash": "2529cdd1a9e8d4d1339679d4bd6b727d2110b2bf0b1a6b21f5a447ab62d10b91", "size": 3918, "status": "c"}, "model/utils/pdb_reres.py|1785482345.0|5821": {"hash": "56e828c67312e9ee1eae28dd45617dbdf0154499399d1c045eeb0ae30ff0ffc6", "size": 5821, "status": "c"}, "scripts/chainsaw_submit.ucl_cs.sh|1787815557.0|2848": {"hash": "a4d715103eff7990f91b86620b196c8dafcfcba8781c682c725052c0a3b8fce9", "size": 2848, "status": "c"}, "scripts/development/data_and_benchmarking/README.md|1787815560.0|759": {"hash": "7ea57716facfba0e62c12e13a06a1fd2cf756ebb2726d2cb8af2eec2ac52dfd1", "size": 759, "status": "c"}, "scripts/development/experiments/chop_newfolds_cluster.py|1785482344.0|3319": {"hash": "9cf4ca18a73970d579907a80833433236e90a9ca8728642c68d5c0a707c0adeb", "size": 3319, "status": "c"}, "scripts/development/experiments/speed_test_human200.py|1785482344.0|1911": {"hash": "c612548e4c733634f5145958843c46abd2c6ab0eed8f3b1980ef71e1329501f5", "size": 1911, "status": "c"}, "scripts/development/data_and_benchmarking/chainsaw_cath1363_train_test_splits.json|1785482344.0|651867": {"hash": "e0f364afb451f2469174d207dd2e348fab994f230b033f4a365ffecf4de79c37", "size": 651867, "status": "c"}, "scripts/development/data_and_benchmarking/chainsaw_model_v3_on_cath1363_test.csv|1785482344.0|220976": {"hash": "882091cbc2d96fc28ddb9ac6c02dbc981585964aced89eb706c5f720a9496e0d", "size": 220976, "status": "c"}, "scripts/development/tests/e2e/test_stride_error.py|1787815562.0|1824": {"hash": "ea81b547c37a1212be0233331c78f50b468e576e122c69964eb7019db398aceb", "size": 1824, "status": "c"}, "scripts/development/tests/e2e/test_basic_usage.py|1787815561.0|3541": {"hash": "6e6726f41fd30b09a1f453db520dd08c5c8c0c0befcc7dc3a8ebac852a947a0b", "size": 3541, "status": "c"}, "scripts/development/notebooks/InteractiveUsageExample.ipynb|1787814459.0|72859": {"hash": "8d80edf5e13f67fe4d0753580fd394c3b447f8c888ba32ef35c57a700ff0e31b", "size": 72859, "status": "c"}, "scripts/development/tests/fixtures/4wgvC.pdb|1785482345.0|204357": {"hash": "42a646c7f37bc4884ce998eed7e1c5536ba023fea728d38c7da3fbf864813733", "size": 204357, "status": "c"}, "scripts/development/tests/fixtures/5yclA.pdb|1785482345.0|66679": {"hash": "1ee3e6b862eb7876d82f538eb60f416be2497e9a694e179515aaa60422c472d6", "size": 66679, "status": "c"}, "scripts/development/tests/unit/__init__.py|1787814020.0|117": {"hash": "d2258c1c2758ec7561f816ee2101b28adea7806b011b72c6f81abb1aafd2b63c", "size": 117, "status": "c"}, "scripts/development/tests/fixtures/AF-A0A0A9LVA1-F1-model_v4.pdb|1785482345.0|9558": {"hash": "bf693d9fc876e48afdc27669105217041e1056f066b0929e7170149ae34e06ea", "size": 9558, "status": "c"}, "scripts/development/tests/fixtures/AF-A0A1W2PQ64-F1-model_v4.pdb|1785482345.0|132353": {"hash": "882619402336796e5a52e3f39cfbe764454cd124f2612c671af89abc9106c86e", "size": 132353, "status": "c"}, "scripts/development/tests/unit/test_get_predictions.py|1787815564.0|2032": {"hash": "b0dd810f1d8bb43558265387d7bd427bc73d09736fafdd14b13460cc059c66fd", "size": 2032, "status": "c"}, "scripts/development/tests/unit/test_model_results.py|1787814023.0|545": {"hash": "7150bdc8ad0bd0d8dd06926ae439ada93dbc887e58a039d02c9cfbea9ecfddcc", "size": 545, "status": "c"}, "scripts/development/tests/unit/test_prediction_result_file.py|1787814024.0|3396": {"hash": "a28ba1316ec038fe9516adcfc2b38c2a3fb659e32f008ca79a9e0c2b61609a56", "size": 3396, "status": "c"}, "scripts/example_files/AF-A0A1W2PQ64-F1-model_v4.pdb|1785482344.0|132353": {"hash": "882619402336796e5a52e3f39cfbe764454cd124f2612c671af89abc9106c86e", "size": 132353, "status": "c"}, "scripts/development/tests/unit/test_write_results.py|1787815565.0|1254": {"hash": "f550baed5b8ae17e4c4cb28ac1c75e0fcf7e36a07acbf7acc2417cdcfd7fe912", "size": 1254, "status": "c"}, "scripts/get_predictions.py|1787813988.0|15261": {"hash": "e6b9a52c834ab2bb956d2451192571b7fcddd6ecbf5157af778e285b219b65ab", "size": 15261, "status": "c"}, "scripts/ss_mod_chainsaw_submit.ucl_cs.sh|1787815559.0|2717": {"hash": "f1113903466183f13d31f22f55fc4b71a4d205f26779bb9f36021d81a6c96560", "size": 2717, "status": "c"}, "scripts/qsub_ss_mod_chainsaw_submit.ucl_cs.qsub.sh|1785482345.0|385": {"hash": "7f21819709950e4659a43cb7f8e8155ad68906c1ed0a648d371b98345d8a1e33", "size": 385, "status": "c"}, "scripts/stride/HOW_TO_USER_STRIDE_FOR_CONTACT_ORDER_CALCULATIONS|1037182665.0|202": {"hash": "10dfdc1e94eafd7435d14075450a2e505f6a84f1f19c388e1ff9c0c506e3bfa3", "size": 202, "status": "c"}, "scripts/example_files/AF-Q5T5X7-F1-model_v4.pdb|1785482344.0|547397": {"hash": "01720526b692020d5175d7753a45c08c7169d393e3b8e722c4d77875ed9814b8", "size": 547397, "status": "c"}, "scripts/setup.sh|1787815556.0|1246": {"hash": "c40eb540da32adcc2bee967bb6b3e5030939bea0638c3925defe475c69d09c73", "size": 1246, "status": "c"}, "scripts/stride/chk_res.c|1035297819.0|825": {"hash": "02cc23e3b0cd7a3a7c5a99a622f3ad5ee10c4881a111a0dd9ac4658bbf4ffda2", "size": 825, "status": "c"}, "scripts/stride/asngener.c|1035297818.0|7801": {"hash": "0438899e50e1f0cf715d5c91731fe8062fcc972978c54061e74f88003f86f2dc", "size": 7801, "status": "c"}, "scripts/stride/README_stride|1787814445.0|670": {"hash": "59d514e9ec11ca2ba48fa258953adc2724d22b4c1acbd29988b18916586a104c", "size": 670, "status": "c"}, "scripts/stride/Makefile|1049803394.0|925": {"hash": "0efb55ccd799f2ae7632b69e0ef6ef1d8c4b92c541c2cef1ba430fefc96fe88f", "size": 925, "status": "c"}, "scripts/stride/area.c|1035297818.0|1600": {"hash": "ee285275d48379741aeced57e3afb00b024cdb727fc6532790de340b898b0532", "size": 1600, "status": "c"}, "scripts/stride/chk_atom.c|1035297818.0|984": {"hash": "21999523fd68a670d7cade5e655d97cb72496dc98015ebe9916ede5b6d1b56c6", "size": 984, "status": "c"}, "scripts/stride/chkchain.c|1035297819.0|6000": {"hash": "49d857f08256a879fdbeac3144596963bed8de9f34bd0504829cb14266e56382", "size": 6000, "status": "c"}, "scripts/stride/command.c|1035297819.0|1249": {"hash": "eafacda54f66755faf348de51f885f5f33f650bd13e59e5f5cce8a78d9aa9595", "size": 1249, "status": "c"}, "scripts/stride/contact_map.c|1049804488.0|787": {"hash": "3555165f745053062ee2fc0c0c5ff8ee7081eabf01aa864904bd4d8af0a468b6", "size": 787, "status": "c"}, "scripts/stride/contact_order.c|1035378228.0|1934": {"hash": "bd6c6a6796ffc03eaae35c2e3e416a969b8b033432c571f85f4d65a8195b22b8", "size": 1934, "status": "c"}, "scripts/stride/doc/notes.mac|1037181310.0|2430": {"hash": "fb80241a3c68c28f783f2540dca9a0082cb6cc4911cc9cc00a98a94ca11aef07", "size": 2430, "status": "c"}, "scripts/stride/doc/stride.doc|1037181309.0|13219": {"hash": "7a7d2ffb221d2be3de7727c342a746c36d94769fa45b4b49be43a8c068bb759a", "size": 13219, "status": "c"}, "scripts/stride/die.c|1035297819.0|188": {"hash": "74c2d8e5e80663e2e84c227835d768b7d33cf94117736c3b0e4d5bace6484926", "size": 188, "status": "c"}, "scripts/stride/doc/relnotes.doc|1037181309.0|1212": {"hash": "56b34ed0e39b98df78d9a7acf858d31eec4c2c2065824f6642f0a57c6259f0b4", "size": 1212, "status": "c"}, "scripts/stride/doc/notes.dos|1037181310.0|570": {"hash": "0fb128c77d6131fd9476fc00156b4f20b1aaafaf660955ce31538a47c3a8a3c7", "size": 570, "status": "c"}, "scripts/stride/dssp.c|1035297819.0|7785": {"hash": "b93ccc30472c27d733e4aab3e15be9adce37e4d6c2dd4ab0ef073c0c6db2e91c", "size": 7785, "status": "c"}, "scripts/stride/elem.c|1035297819.0|4675": {"hash": "17664a1f184ef6f424f68f1de80c004744e60413c12f3481f2ffdd5120f2d6dc", "size": 4675, "status": "c"}, "scripts/stride/escape.c|1035297819.0|182": {"hash": "2f14f876bf7069e7bba869bdfff219e893a15b90e196791c14aeb20921252a24", "size": 182, "status": "c"}, "scripts/stride/fillasn.c|1035297819.0|17051": {"hash": "9dcaf7d638312e7d0c38f52924bd6443906f86beeb734b318931a1d64aa6e950", "size": 17051, "status": "c"}, "scripts/stride/hbenergy.c|1035297819.0|1981": {"hash": "af086801284711d9a3bad271a499a58d5a553b3efeacd36ff40185c681d2d36f", "size": 1981, "status": "c"}, "scripts/stride/initchn.c|1035297820.0|1616": {"hash": "95083f51264540a8b0cc3ab982063201ed5b94355f75cc81fe9bb936832285af", "size": 1616, "status": "c"}, "scripts/stride/filename.c|1035297819.0|522": {"hash": "cbfc56f86386e195a967590ebceb30ba3fe2763b743b4e4750e70c560b1b601d", "size": 522, "status": "c"}, "scripts/stride/geometry.c|1035305053.0|14030": {"hash": "e025c3d64bece4a94da6cc23a1e880223926462c34ebf824bb8af30cb7e8a76e", "size": 14030, "status": "c"}, "scripts/stride/hydrbond.c|1044630447.0|19373": {"hash": "0ae7e41e4705aefa131045de9894484d6869aed78d8841b3e68f3c462bd51376", "size": 19373, "status": "c"}, "scripts/stride/helix.c|1035297819.0|3126": {"hash": "6ee65307867be8c9351fb3349d0cee9d583cded15cec2307779815952e12e7ce", "size": 3126, "status": "c"}, "scripts/stride/measure.c|1035297820.0|9578": {"hash": "fd65acb9116a2055fea8a53c26acf4aaf123b7ce9ae6ad5044b003566990a3c9", "size": 9578, "status": "c"}, "scripts/stride/mergepat.c|1035297822.0|15433": {"hash": "0c72e01b8283971c758a55039e97d3e9b97cab519066d439b433abec832db181", "size": 15433, "status": "c"}, "scripts/stride/memory.c|1035297820.0|3448": {"hash": "3b00d3d6d9b3e475427f586d6aa42f761e37542b26e8bbb226bbe1d97f2d57d7", "size": 3448, "status": "c"}, "scripts/stride/nsc.c|1035297822.0|28965": {"hash": "a8af9d77e7554b8bee70375ea08be6bfa741192b827f4e5b0201bb5a583adbb5", "size": 28965, "status": "c"}, "scripts/stride/molscr.c|1035297822.0|2138": {"hash": "aaea89e67ba7d368dde129b31968b6c07f53f45829a9816eef980487b28b55cc", "size": 2138, "status": "c"}, "scripts/stride/nsc.h|1035297822.0|5697": {"hash": "7ea7ec08b2faa5b5b2608e8156d6ee854f83c1ecf8f8436ecd1747bb0f16a682", "size": 5697, "status": "c"}, "scripts/stride/p_atom.c|1035297823.0|2764": {"hash": "857286c25dbf21debeeaec0e504535916984b0412f98b1c70a2e308c8d89d4bd", "size": 2764, "status": "c"}, "scripts/stride/p_endmdl.c|1035297823.0|189": {"hash": "44e5584b7f07e519cfe43438a654547201ac892f846ceec0f7907d7d38bcced5", "size": 189, "status": "c"}, "scripts/stride/p_jrnl.c|1035297823.0|318": {"hash": "af3f53b6f485ec0714f102450f73806dfa570b5c13d7e0f041daccb720568825", "size": 318, "status": "c"}, "scripts/stride/p_helix.c|1035297823.0|1163": {"hash": "052760462ca9a990af93f915ad3356a3fd4b130546563f930df7101176df9201", "size": 1163, "status": "c"}, "scripts/stride/p_compnd.c|1035297823.0|236": {"hash": "617256e2c91b1932a1bb086673a784845c6393b3c7b9dd611ba11cce9fd4da47", "size": 236, "status": "c"}, "scripts/stride/p_expdta.c|1035297823.0|381": {"hash": "f3d034be520b7e92ac138b4ca9cba76a42b261c1c188ad50e9881d094429056f", "size": 381, "status": "c"}, "scripts/stride/p_model.c|1035297823.0|133": {"hash": "948c57ce4029e64b2000df75d524081a3984b7d7c1dbb85379cce0a38d74c65a", "size": 133, "status": "c"}, "scripts/stride/one2thr.c|1035297822.0|986": {"hash": "a1aae2c370ac6b8b5d244d81f44ca257e8331d06a6a48efc3335dd8a2b86ab49", "size": 986, "status": "c"}, "scripts/stride/p_sheet.c|1035297823.0|2875": {"hash": "f137e9b1e73e228322e1a94476172b77b0e3bb0745be28ffaa9cb908663f65bf", "size": 2875, "status": "c"}, "scripts/stride/p_ssbond.c|1035297823.0|979": {"hash": "279cafac1c7d0f07a40a2ed55d2ba2a63c6c838f4e25a64c7ac46e0fbad2cfd7", "size": 979, "status": "c"}, "scripts/stride/p_turn.c|1035297824.0|1046": {"hash": "ca0cd408b570aed9959af78cef6017e82267b1ec8a591c3976f4a648da59f4ea", "size": 1046, "status": "c"}, "scripts/stride/pdb1res.pdb|1044971484.0|62937": {"hash": "d9c33efa05bf1a39dbaf576a6e034eaee1a8194b686f106cf207ca104f4465f4", "size": 62937, "status": "c"}, "scripts/stride/pdbasn.c|1035297824.0|1596": {"hash": "69d499b32c25069790c64b23101ca7b49d798ec77263b65b24d080698f6fb435", "size": 1596, "status": "c"}, "scripts/stride/p_rem.c|1035297823.0|960": {"hash": "7f2324af196b9a78f43540e55417643af6425d849d7904041712cf8a1568a554", "size": 960, "status": "c"}, "scripts/stride/outseq.c|1035297822.0|751": {"hash": "7c8c97b675cd6604f604a648791e43b0954f6bfc7d313cf66d2abac0b4feb6d5", "size": 751, "status": "c"}, "scripts/stride/pdb2btm.pdb|1044971468.0|347166": {"hash": "eeee8a49a5078c955a17346f402d8d28124b51e45618dba52f83c00f180352b5", "size": 347166, "status": "c"}, "scripts/stride/pdb1fmc.pdb|1044971490.0|377622": {"hash": "5e8efb8bce02af061a2be14eb4f15118422506aa4cd4406083fdd7f5377017cb", "size": 377622, "status": "c"}, "scripts/stride/phipsi.c|1035297824.0|1362": {"hash": "01519c52a8f8512930b2ad8685d7b74ce473b997d19bf1096f34eef3338ed8f0", "size": 1362, "status": "c"}, "scripts/stride/place_h.c|1035297824.0|1133": {"hash": "9f2a529f561d88916030cb3a668d46af5be7a2e94936f56863de5ec921591267", "size": 1133, "status": "c"}, "scripts/stride/rdmap.c|1035297824.0|11643": {"hash": "df41c31438be1c2763a6d46774f18a5101626b0e82d0e6dfe8ed2135a88df145", "size": 11643, "status": "c"}, "scripts/stride/rdpdb.c|1035297824.0|3568": {"hash": "a4c9f7c73247c05294235f220f2d8215257fc511a26e25357bec66861f68ef1f", "size": 3568, "status": "c"}, "scripts/stride/report.c|1226913206.0|10236": {"hash": "48d13dda3b1e4e95c1776f82698c2c0b850bec25b6e1c4e118446f9ccaacf27c", "size": 10236, "status": "c"}, "scripts/stride/pdb1mjh.pdb|1044971741.0|247779": {"hash": "8881b97b8e05043608cf050a4657d32f3723879c50ae2355c6338fe2871cc65f", "size": 247779, "status": "c"}, "scripts/stride/protot.h|1049803446.0|10132": {"hash": "a06bf6616ddedeb269af2bdc0af84a515751d3070cb07141d67cbbda7c5a7580", "size": 10132, "status": "c"}, "scripts/stride/sheet.c|1035297825.0|15123": {"hash": "a115a7af69004a26ab234da32d0438f0f4236b0de4386240b8fde570b0026b4c", "size": 15123, "status": "c"}, "scripts/stride/splitstr.c|1035297825.0|981": {"hash": "d7206158c3de524844cc2c3104cfbe07ed1ae504a1ea28836ce662f6851fe78d", "size": 981, "status": "c"}, "scripts/stride/ssbond.c|1035297826.0|1891": {"hash": "eebae52935d4a558dd633bb0c1c2f1c415f8d99f876a1a9772b7d4366bdb101d", "size": 1891, "status": "c"}, "scripts/stride/stred.c|1035297826.0|581": {"hash": "dbfcb177e4888c4d2e836a75503aa05e4102bbed7397681407b95d9ba8002460", "size": 581, "status": "c"}, "scripts/stride/stride|1785483222.0|170072": {"hash": "31db248eb0890d0b958e362f386a31f229b671a8430b422bf70ddf60191469ff", "size": 170072, "status": "c"}, "scripts/stride/stride.c|1049803551.0|10286": {"hash": "64377dc2f8695ff468f811e51fb1a1513750211d2500be45fd0e0719bab0fa9c", "size": 10286, "status": "c"}, "scripts/stride/strutil.c|1035297828.0|8851": {"hash": "065ab47823715976c773b4ceea2bea3ea109a9b00a7085dd9cc2c3998e76e954", "size": 8851, "status": "c"}, "scripts/stride/stride.h|1049803421.0|9106": {"hash": "6b162fdfe6e63c03ace5bf4fe8be964db44207866487d59dc8856a01a49eae14", "size": 9106, "status": "c"}, "scripts/stride/thr2one.c|1035297828.0|1132": {"hash": "8643f78cd9a937f7ddf6ef2c0197f326b28ae8d8048401a383e3b266b2b00880", "size": 1132, "status": "c"}, "scripts/stride/stride.tgz|1785482345.0|316993": {"hash": "d7481f3c84375af7381f11decd62227fc9dd253e074edabd23f9f6ca77769c54", "size": 316993, "status": "c"}, "scripts/stride/turn.c|1035297829.0|4204": {"hash": "a484c24f0cb975cc9d2a3b7c51911a29de1dfe122bdaec1451a92d7a1f8b4b0b", "size": 4204, "status": "c"}, "scripts/test_zip_extract.py|1785482345.0|2094": {"hash": "0377d6abc85b48f20b417edd7de672a8e90a51fe103b7f92f5e3f4bb776aaf9a", "size": 2094, "status": "c"}, "scripts/stride/tolostr.c|1035297828.0|360": {"hash": "f2576e1e83f6f9d8bf65f13d3b56f71962227fa09fb6cecd78955b8b2b1c19ff", "size": 360, "status": "c"}, "scripts/zip_extract.py|1785482345.0|1743": {"hash": "8b0f33c8ccbfacc2960cd715eae18422d6b7664fce1c6ab16f3c8a5a8151145e", "size": 1743, "status": "c"}, "weight/model_v1/best_val_metrics.json|1785482345.0|1786": {"hash": "1c9524288ae797faf43f4c82973ad91af89f75c774be5f3fdc633e6264a5aff4", "size": 1786, "status": "c"}, "weight/model_v2/best_val_metrics.json|1785482345.0|2603": {"hash": "bbaa568a993274857ad33dd7069ff257d24480c1861b89086f10fb9b5f892ae0", "size": 2603, "status": "c"}, "weight/model_v1/weights.pt|1785482345.0|2396293": {"hash": "6e8b9459f66bf2af04f652083e8b1991bf870f368f424691ae48a8445c09ba37", "size": 2396293, "status": "c"}, "weight/model_v2/chains.json|1785482345.0|2977860": {"hash": "d6b78b238cd00ed0d71da10e37e592cff19c59085217a0cc78898e74d30139f9", "size": 2977860, "status": "c"}, "weight/model_v2/weights.pt|1785482345.0|2396549": {"hash": "1fca05a2a64a80990e7dfb6013c26fe00c53fd78b39d85ab3c9973939781b183", "size": 2396549, "status": "c"}, "weight/model_v3/weights.pt|1785482345.0|2396726": {"hash": "f21d5451e285e347582944e4017822379c31866d343593d71bccc8eb31ff9c9e", "size": 2396726, "status": "c"}, "weight/model_v1/chains.json|1785482345.0|2833360": {"hash": "5b4711a08e97a8de716fe093a66cca523071a5d49477d2fd7c32d25eb4dfe585", "size": 2833360, "status": "c"}}} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9ea7d7a769ab087e48aed21ab599105d33675570 --- /dev/null +++ b/README.md @@ -0,0 +1,221 @@ +--- +frameworks: +- PyTorch +language: +- en +license: mit +tags: +- OneScience +- bioscience +- protein-domain-prediction +- protein-structure-analysis +- Chainsaw +tasks: [] +datasets: [] +--- + +

+ + Chainsaw + +

+ +# Model Introduction + +Chainsaw is a fully convolutional neural network designed to predict protein domain boundaries from three-dimensional protein structures. The model extracts residue-distance features and STRIDE secondary-structure features from PDB or mmCIF structures, predicts the probability that residue pairs belong to the same structural domain, and then applies post-processing to generate the final domain segmentation. + +Paper: [Chainsaw: protein domain segmentation with fully convolutional neural networks](https://doi.org/10.1093/bioinformatics/btae296) + +# Model Description + +Chainsaw takes a three-dimensional protein structure as input rather than an amino acid sequence alone. Its main processing pipeline is as follows: + +1. Parse the specified protein chain and generate a residue-distance matrix; +2. Run STRIDE to calculate secondary-structure features; +3. Use a fully convolutional neural network to predict residue-pair domain co-membership; +4. Convert the predicted matrix into continuous or discontinuous domain boundaries and output confidence scores. + +The model package includes three versions of the official pretrained weights, with `model_v3` used by default. Basic inference does not require downloading additional weights or retraining the model. + +# Use Cases + +| Use Case | Description | +| :---: | :--- | +| Single-structure prediction | Segment a protein chain from a PDB or mmCIF file into structural domains. | +| Batch structure prediction | Process multiple PDB or mmCIF structure files in a directory. | +| AlphaFold structure analysis | Identify domain boundaries in AlphaFold-predicted structures. | +| Domain boundary screening | Output the number of domains, residue ranges, confidence scores, and runtime for downstream analysis. | + +# Usage + +## 1. OneCode + +You can use the OneCode online environment for an intelligent one-click AI4S programming experience: + +[Try OneCode for AI4S Programming](https://web-2069360198568017922-iaaj.ksai.scnet.cn:58043/home) + +## 2. Manual Installation + +**Hardware Requirements** + +- Supports inference on both CPU and DCU; +- STRIDE, structure parsing, and part of the post-processing pipeline are executed on CPU. + +### Download the Model Package + +Install the Hugging Face command-line tool and download the model repository: + +```bash +pip install -U huggingface_hub + +hf download OneScience-Group/Chainsaw --local-dir ./Chainsaw +cd Chainsaw +``` + +### Install the Runtime Environment + +**DCU Environment** + +```bash +conda create -n onescience311 python=3.11 -y +conda activate onescience311 + +pip install onescience[bio-dcu] \ + -i http://mirrors.onescience.ai:3141/pypi/simple/ \ + --trusted-host mirrors.onescience.ai +``` + +### Compile STRIDE + +Chainsaw relies on STRIDE to generate secondary-structure features. The STRIDE source code is located in `scripts/stride/`. Before the first run, execute: + +```bash +cd scripts/stride +make +chmod +x stride +cd ../.. +``` + +By default, the inference script looks for the STRIDE executable at: + +```text +scripts/stride/stride +``` + +When running commands from the root directory of the Hugging Face model package, no additional path configuration is required. + +If you need to specify the STRIDE executable explicitly, use: + +```bash +export STRIDE_EXE=scripts/stride/stride +``` + +### Weights and Data Preparation + +The Hugging Face model package already contains the official weights and configuration files required for basic inference. No additional model weights need to be downloaded at runtime. + +| Model Version | Weights | Configuration | +| --- | --- | --- | +| model_v1 | `weight/model_v1/weights.pt` | `conf/model_v1/` | +| model_v2 | `weight/model_v2/weights.pt` | `conf/model_v2/` | +| model_v3 (default) | `weight/model_v3/weights.pt` | `conf/model_v3/` | + +Input structures must be provided by the user. Both PDB and mmCIF files are supported. + +The example files included in: + +```text +scripts/example_files/ +``` + +can be used for basic validation without downloading any training dataset. + +### Quick Inference + +All commands below should be executed from the root directory of the downloaded Hugging Face model package. + +If the output directory does not exist, the inference script will create it automatically. + +**DCU Inference** + +```bash +unset CUDA_VISIBLE_DEVICES +export TORCHDYNAMO_DISABLE=1 + +python scripts/get_predictions.py \ + --structure_file scripts/example_files/AF-A0A1W2PQ64-F1-model_v4.pdb \ + --output output/inference/predictions_dcu.tsv +``` + +**CPU Inference** + +```bash +export CUDA_VISIBLE_DEVICES="" +export TORCHDYNAMO_DISABLE=1 + +python scripts/get_predictions.py \ + --structure_file scripts/example_files/AF-A0A1W2PQ64-F1-model_v4.pdb \ + --output output/inference/predictions_cpu.tsv +``` + +**Batch Processing of Structure Files** + +Place the structures to be predicted in: + +```text +input/structures/ +``` + +Then run: + +```bash +python scripts/get_predictions.py \ + --structure_directory input/structures \ + --output output/inference/batch_predictions.tsv +``` + +**Using Another Official Model Version** + +```bash +python scripts/get_predictions.py \ + --model_dir weight/model_v1 \ + --config_dir conf/model_v1 \ + --structure_file input/protein.pdb \ + --output output/inference/model_v1_predictions.tsv +``` + +### Output + +The inference result is saved as a TSV file, with one row corresponding to each input structure chain. + +| Field | Description | +| --- | --- | +| `chain_id` | Identifier of the input protein chain. If no chain is explicitly specified, the first available chain is used. | +| `sequence_md5` | MD5 hash of the input amino acid sequence. | +| `nres` | Number of valid residues. | +| `ndom` | Number of predicted structural domains. | +| `chopping` | Predicted domain residue ranges. Underscores connect fragments belonging to the same discontinuous domain, while commas separate different domains. | +| `confidence` | Prediction confidence score. | +| `time_sec` | Inference time for a single structure, in seconds. | + +### Training + +The upstream repository does not publicly provide the complete training dataset, an executable end-to-end training entry point, or the full set of training hyperparameters such as learning rate, batch size, and number of epochs. + +Therefore, this Hugging Face model package does not provide a training command. + +The official pretrained weights included in the package can be used directly for the intended inference tasks. + +# OneScience Official Resources + +| Platform | OneScience Main Repository | Skills Repository | +| --- | --- | --- | +| Gitee | https://gitee.com/onescience-ai/onescience | https://gitee.com/onescience-ai/oneskills | +| GitHub | https://github.com/onescience-ai/OneScience | https://github.com/onescience-ai/oneskills | + +# Citation and License + +- Paper: [Chainsaw: protein domain segmentation with fully convolutional neural networks](https://doi.org/10.1093/bioinformatics/btae296) +- Official implementation: https://github.com/JudeWells/chainsaw +- Chainsaw is distributed under the MIT License. +- STRIDE and other third-party components are subject to their respective original copyright notices, licenses, and terms of use. \ No newline at end of file diff --git a/conf/LICENSE b/conf/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..d47d4551e45a353b890a51ef2e4e0740274aea19 --- /dev/null +++ b/conf/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Jude Wells + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/conf/config.json b/conf/config.json new file mode 100644 index 0000000000000000000000000000000000000000..17f1293feb0e6c01e4dae7d2dfac5a013d39df2f --- /dev/null +++ b/conf/config.json @@ -0,0 +1,12 @@ +{ + "model_version": "model_v3", + "model_dir": "weight/model_v3", + "model_config_dir": "conf/model_v3", + "example_structure": "scripts/example_files/AF-A0A1W2PQ64-F1-model_v4.pdb", + "output": "output/inference/predictions.tsv", + "device": "auto", + "post_process_domains": true, + "remove_disordered_domain_threshold": 0.35, + "min_domain_length": 30, + "min_ss_components": 2 +} diff --git a/conf/model_v1/config.json b/conf/model_v1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5549015593d43bff111d1bacc172f3fa46ee9f10 --- /dev/null +++ b/conf/model_v1/config.json @@ -0,0 +1,56 @@ +{ + "experiment_name": "secondary_structure", + "experiment_group": "filtered_splits", + "lr": 0.0005, + "weight_decay": 0.001, + "val_freq": 1, + "epochs": 80, + "accumulation_steps": 16, + "data": { + "crop_size": null, + "crop_type": null, + "batch_size": 1, + "feature_dir": "/mnt/disk2/domdet/features/nopae/2d_features_ss", + "label_dir": "/mnt/disk2/domdet/features/nopae/pairwise", + "chains_csv": null, + "evaluate_test": false, + "eval_casp_10_plus": false, + "splits_file": "splits_multi.json", + "validation_splits": [ + "filt_val", + "filt_val_multi", + "filt_val_many" + ], + "remove_missing_residues": false, + "using_alphafold_features": false, + "recycling": true, + "add_padding_mask": false, + "training_exclusion_json": null, + "multi_proportion": 0.65, + "train_ids": null, + "exclude_cath_test": false + }, + "learner": { + "model": { + "type": "trrosetta", + "kwargs": { + "filters": 32, + "kernel": 3, + "num_layers": 31, + "in_channels": 5 + } + }, + "assignment": { + "type": "sparse_lowrank", + "kwargs": { + "N_iters": 3, + "K_init": 4, + "linker_threshold": 20 + } + }, + "max_recycles": 1, + "save_val_best": true, + "x_has_padding_mask": false + }, + "num_trainable_params": 577889 +} \ No newline at end of file diff --git a/conf/model_v1/feature_config.json b/conf/model_v1/feature_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e52e2618dd250819261810a75d489e23117ef68d --- /dev/null +++ b/conf/model_v1/feature_config.json @@ -0,0 +1,9 @@ +{ + "description": "alpha distance only, keep the start and end boundaries in one channel", + "alpha_distances": true, + "beta_distances": false, + "ss_bounds": false, + "negative_ss_end": false, + "separate_channel_ss_start_end": false, + "same_channel_boundaries_and_ss": true +} \ No newline at end of file diff --git a/conf/model_v2/config.json b/conf/model_v2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e014876d4aa89fc297d78775594c96a6ede8911a --- /dev/null +++ b/conf/model_v2/config.json @@ -0,0 +1,60 @@ +{ + "experiment_name": "ss_c_base_no_excl", + "experiment_group": "ss_mod", + "lr": 0.0001, + "weight_decay": 0.001, + "val_freq": 1, + "epochs": 30, + "accumulation_steps": 16, + "data": { + "crop_size": null, + "crop_type": null, + "batch_size": 1, + "feature_dir": "/mnt/disk2/domdet/features/nopae/2d_features_ss_mod", + "label_dir": "/mnt/disk2/domdet/features/nopae/pairwise", + "chains_csv": null, + "evaluate_test": true, + "eval_casp_10_plus": false, + "splits_file": "splits_multi.json", + "validation_splits": [ + "filt_val", + "filt_val_multi", + "filt_val_many" + ], + "remove_missing_residues": false, + "using_alphafold_features": false, + "recycling": true, + "add_padding_mask": false, + "training_exclusion_json": null, + "multi_proportion": 0.65, + "train_ids": null, + "exclude_cath_test": false, + "start_end_ss": false + }, + "learner": { + "uncertainty_model": false, + "save_every_epoch": true, + "model": { + "type": "trrosetta", + "kwargs": { + "filters": 32, + "kernel": 3, + "num_layers": 31, + "in_channels": 7, + "symmetrise_output": true + } + }, + "assignment": { + "type": "sparse_lowrank", + "kwargs": { + "N_iters": 3, + "K_init": 4, + "linker_threshold": 30 + } + }, + "max_recycles": 1, + "save_val_best": true, + "x_has_padding_mask": false + }, + "num_trainable_params": 577953 +} \ No newline at end of file diff --git a/conf/model_v2/feature_config.json b/conf/model_v2/feature_config.json new file mode 100644 index 0000000000000000000000000000000000000000..0b60aeb1fd147f6a0e0b86c906d090851bede792 --- /dev/null +++ b/conf/model_v2/feature_config.json @@ -0,0 +1,9 @@ +{ + "description": "alpha distance only, keep the start and end boundaries in same channel as co-membership", + "alpha_distances": true, + "beta_distances": false, + "ss_bounds": true, + "negative_ss_end": false, + "separate_channel_ss_start_end": false, + "same_channel_boundaries_and_ss": false +} \ No newline at end of file diff --git a/conf/model_v3/config.json b/conf/model_v3/config.json new file mode 100644 index 0000000000000000000000000000000000000000..8842d6c9a90333192d21984fd25b45fd267c9534 --- /dev/null +++ b/conf/model_v3/config.json @@ -0,0 +1,68 @@ +{ + "experiment_name": "C_mul65_do30", + "experiment_group": "cath_new", + "lr": 0.0002, + "weight_decay": 0.001, + "val_freq": 1, + "epochs": 15, + "lr_scheduler": { + "type": "exponential", + "gamma": 0.9 + }, + "accumulation_steps": 16, + "data": { + "splits_file": "splits_new_cath_featurized.json", + "validation_splits": [ + "validation", + "test" + ], + "crop_size": null, + "crop_type": null, + "batch_size": 1, + "feature_dir": "../features/new_cath/2d_features", + "label_dir": "../features/new_cath/pairwise", + "chains_csv": null, + "evaluate_test": false, + "eval_casp_10_plus": false, + "remove_missing_residues": false, + "using_alphafold_features": false, + "recycling": false, + "add_padding_mask": false, + "training_exclusion_json": null, + "multi_proportion": 0.65, + "train_ids": "splits_new_cath_featurized.json", + "exclude_test_topology": false, + "cluster_sampling_training": true, + "dist_transform": "unidoc_exponent", + "distance_denominator": 10, + "merizo_train_data": false, + "redundancy_level": "S60_comb" + }, + "learner": { + "uncertainty_model": false, + "save_every_epoch": true, + "model": { + "type": "trrosetta", + "kwargs": { + "filters": 32, + "kernel": 3, + "num_layers": 31, + "in_channels": 5, + "dropout": 0.3, + "symmetrise_output": true + } + }, + "assignment": { + "type": "sparse_lowrank", + "kwargs": { + "N_iters": 3, + "K_init": 4, + "linker_threshold": 30 + } + }, + "max_recycles": 0, + "save_val_best": true, + "x_has_padding_mask": false + }, + "num_trainable_params": 577889 +} \ No newline at end of file diff --git a/conf/model_v3/feature_config.json b/conf/model_v3/feature_config.json new file mode 100644 index 0000000000000000000000000000000000000000..4accf82d23a462047b2d18cacdb770a7f9fc3cf2 --- /dev/null +++ b/conf/model_v3/feature_config.json @@ -0,0 +1,9 @@ +{ + "description": "alpha distance only, keep the start and end boundaries in one channel but use -1", + "alpha_distances": true, + "beta_distances": false, + "ss_bounds": true, + "negative_ss_end": true, + "separate_channel_ss_start_end": false, + "same_channel_boundaries_and_ss": false +} \ No newline at end of file diff --git a/conf/pyproject.toml b/conf/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..6faf54859adcb78f1796226b0d3d9aefb31bcd30 --- /dev/null +++ b/conf/pyproject.toml @@ -0,0 +1,2 @@ +[tool.ruff] +line-length = 120 \ No newline at end of file diff --git a/conf/requirements.txt b/conf/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..37778dc199d88f8a4322986fa9e9baab83e85e71 --- /dev/null +++ b/conf/requirements.txt @@ -0,0 +1,7 @@ +# PyTorch is supplied by the SCNet DTK environment and must not be replaced +# by the upstream torch==2.0.1 pin. +biopython>=1.81,<2 +numpy>=1.24,<2 +pandas>=2.0 +pydantic>=1.10,<3 +scipy>=1.10,<2 diff --git a/conf/requirements_to_freeze.dev.txt b/conf/requirements_to_freeze.dev.txt new file mode 100644 index 0000000000000000000000000000000000000000..55b033e901cdda93a26ac64b418f260224260a39 --- /dev/null +++ b/conf/requirements_to_freeze.dev.txt @@ -0,0 +1 @@ +pytest \ No newline at end of file diff --git a/conf/requirements_to_freeze.txt b/conf/requirements_to_freeze.txt new file mode 100644 index 0000000000000000000000000000000000000000..cbe033d4a96c9ea4236098b1df7570e74360ec3d --- /dev/null +++ b/conf/requirements_to_freeze.txt @@ -0,0 +1,8 @@ +numpy +pandas +torch +matplotlib +requests +biopython +pydantic +click \ No newline at end of file diff --git a/configuration.json b/configuration.json new file mode 100644 index 0000000000000000000000000000000000000000..fc84d6a7b9806f83c3933b4c76a2122644ea7706 --- /dev/null +++ b/configuration.json @@ -0,0 +1,13 @@ +{ + "model_name": "Chainsaw", + "framework": "PyTorch", + "task": "protein-domain-boundary-prediction", + "entry_points": { + "inference": "scripts/get_predictions.py", + "predict": "scripts/get_predictions.py" + }, + "source_package": "model", + "config": "conf/config.json", + "weight_dir": "weight", + "license": "MIT" +} diff --git a/model/__init__.py b/model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/model/constants.py b/model/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..d82221eab38b0370d8863b637ddb3a972a0b1188 --- /dev/null +++ b/model/constants.py @@ -0,0 +1,30 @@ +from pathlib import Path +import os + + +REPO_ROOT = Path(__file__).parent.parent.resolve() +STRIDE_EXE = os.environ.get('STRIDE_EXE', str(REPO_ROOT / "scripts" / "stride" / "stride")) +PYMOL_EXE = "/Applications/PyMOL.app/Contents/MacOS/PyMOL" # only required if you want to generate 3D images + +_3to1 = { +'ALA': 'A', +'CYS': 'C', +'ASP': 'D', +'GLU': 'E', +'PHE': 'F', +'GLY': 'G', +'HIS': 'H', +'ILE': 'I', +'LYS': 'K', +'LEU': 'L', +'MET': 'M', +'ASN': 'N', +'PRO': 'P', +'GLN': 'Q', +'ARG': 'R', +'SER': 'S', +'THR': 'T', +'VAL': 'V', +'TRP': 'W', +'TYR': 'Y', +} diff --git a/model/domain_assignment/assigners.py b/model/domain_assignment/assigners.py new file mode 100644 index 0000000000000000000000000000000000000000..3a7424f86675a2ff889026c4a13cf143c2acf9cb --- /dev/null +++ b/model/domain_assignment/assigners.py @@ -0,0 +1,76 @@ +import numpy as np +import torch +from model.domain_assignment.sparse_lowrank import greedy_V + + +class BaseAssigner: + + def __call__(self, y_pred, uncertainty=False): + assert y_pred.ndim == 2 + if torch.is_tensor(y_pred): + y_pred = y_pred.detach().cpu().numpy() + return self.assign_domains(y_pred) + + +class SparseLowRank(BaseAssigner): + + """Use Brooks' method to generate a matrix of cluster assignments, + and post-process to remove any excessively small clusters. + """ + + def __init__(self, N_iters=3, K_init=4, linker_threshold=5, cost_type="mse"): + self.N_iters = N_iters + self.K_init = K_init + self.linker_threshold = linker_threshold + self.cost_type = cost_type + + def get_entropy(self, y_pred): + """calculate the entropy of the upper triangle of the matrix + used for uncertainty calculation but does not correlate with NDO as well + as likelihood of assignment under y_pred + """ + return np.triu(-1 * y_pred * np.log(y_pred) - (1 - y_pred) * np.log(1 - y_pred), 1).sum() + + + def assign_domains(self, y_pred): + # N x K, columns are then indicator vectors + epsilon = 1e-6 + y_pred = np.clip(y_pred, epsilon, 1-epsilon) + V, loss = greedy_V(y_pred, N_iters=self.N_iters, K_init=self.K_init, cost_type=self.cost_type) + K = V.shape[-1] + A = V@V.T + average_likelihood = np.exp((A * np.log(y_pred) + (1-(A))*np.log(1-y_pred)).mean()) + # throw away small clusters + V = np.where( + V.sum(0, keepdims=True) < self.linker_threshold, # 1, K + np.zeros_like(V), + V, + ) + + assignments = { + "linker": np.argwhere((V == 0).sum(-1) == K).reshape(-1) + } + + domain_ix = 1 + for col_ix in range(K): + cluster_inds = np.argwhere(V[:, col_ix] == 1).reshape(-1) + if cluster_inds.size > 0: + assignments[f"domain_{domain_ix}"] = cluster_inds + domain_ix += 1 + return assignments, round(average_likelihood, 4) + + +class SpectralClustering(BaseAssigner): + """Challenge is how to determine n_clusters. + + We can look at eigengaps + Or use MSE, running clustering multiple times. + Or use Brooks' method and then refine with SC. + """ + + def __init__(self, n_cluster_method="mse", max_domains=10): + self.n_cluster_method = n_cluster_method + self.max_domains = max_domains # TODO - make this flexible or length dependent. + + def assign_domains(self, y_pred): + raise NotImplementedError() diff --git a/model/domain_assignment/sparse_lowrank.py b/model/domain_assignment/sparse_lowrank.py new file mode 100644 index 0000000000000000000000000000000000000000..0c64ebc221159e9f0df029863df05102d70d0f5f --- /dev/null +++ b/model/domain_assignment/sparse_lowrank.py @@ -0,0 +1,121 @@ +import numpy as np + + +def mse_loss_at_residue(V_hat, Y, residue_index): + # loss contribution of residue given current assignment + # assumes Y is symmetric + d = residue_index + # avoid double-counting the diagonal + return 2*((V_hat[d]@V_hat.T - Y[d])**2).sum() - (V_hat[d]@V_hat[d] - Y[d,d])**2 + + +def null_mse_loss_at_residue(Y, residue_index): + # loss contribution of residue when unassigned + d = residue_index + return 2*(Y[d]**2).sum() - Y[d,d]**2 + + +def mse_loss(V_hat, Y): + # only use to compute initial loss + return ((V_hat@V_hat.T - Y)**2).sum() + + +def nll_loss(V_hat, Y): + """We now interpret ys as probabilities. + + We want to minimise the negative log probability. + + NB Y must be symmetric and [0-1] + """ + # only use to compute initial loss + log_pij = (V_hat@V_hat.T) * np.log(Y) + (1-(V_hat@V_hat.T))*np.log(1-Y) + # simple sum double counts off diagonals but single counts on diagonals, so need + # to add diagonal again before dividing by 2 + return -0.5*(log_pij.sum() + np.diag(log_pij).sum()) + + +def nll_loss_at_residue(V_hat, Y, residue_index): + # d is residue_index + d = residue_index # V_hat[d]@V_hat.T is just Adj without instantiating A + # n.b. whereas in case of mse we double-count off-diagonals, here we don't, hence the difference + log_pdj = (V_hat[d]@V_hat.T) * np.log(Y[d]) + (1-(V_hat[d]@V_hat.T))*np.log(1-Y[d]) + return -log_pdj.sum() + + +def null_nll_loss_at_residue(Y, residue_index): + return - np.log(1-Y[residue_index]).sum() + + +def greedy_V(Y, N_iters=3, K_init=4, cost_type="mse"): + """ + Learn a binary matrix V, with at most one nonzero entry per row, to minimize + + || VV' - Y ||_2 + + This is done by initializing a V of all zeros, and then doing a greedy optimization. + V is initially D x K_init, where D is the number of residues (i.e., where Y is DxD). + K is learned automatically; different values of K_init will not change the result but + might be more or less efficient in terms of memory usage. The code keeps track of the + current number of nonzero columns of V, and when all the columns are full it adds extra + columns of zeros at the end. + + Each iteration sweeps through all residues (i.e. rows of V) once. + + The implementation relies on additivity of loss to reduce computational cost + by only computing required increments to the loss at each iteration. + + + INPUTS: + + Y: matrix of predictions with entries in [0, 1] + N_iters: number of iterations + K_init: initial number of columns of V, adjust this for tweaking performance + """ + Y = (Y + Y.T) / 2 # required for consistency with model_v1 + V_hat = np.zeros((Y.shape[0], K_init), dtype=np.uint8) + if cost_type == "mse": + loss = mse_loss(V_hat, Y) # initial loss for the zero matrix V_hat = 0 + elif cost_type == "nll": + loss = nll_loss(V_hat, Y) + else: + raise ValueError(cost_type) + K_max = K_init # track number of columns in K_max + + for it in range(N_iters): + for d in range(V_hat.shape[0]): + # d is a residue index + # compute the loss, excluding contribution of this residue + # TODO check minus signs are consistent + loss_minus_d = loss - mse_loss_at_residue(V_hat, Y, d) + + # sweep through all K+1 options and compute what the contribution to the loss would be + V_hat[d] *= 0 + + # loss with no assignment + L0 = loss_minus_d + null_mse_loss_at_residue(Y, d) + L_opt = np.zeros(K_max) + for k in range(K_max): + # note this could be vectorized with a bit of work + V_hat[d,k] = 1 + L_opt[k] = loss_minus_d + mse_loss_at_residue(V_hat, Y, d) + V_hat[d,k] = 0 + + # select the option which minimizes the squared error + z = np.argmin(L_opt) + if L_opt[z] < L0: + V_hat[d,z] = 1 + + # update loss + loss = loss_minus_d + mse_loss_at_residue(V_hat, Y, d) + + if z == K_max-1: + # Expand V_hat to make room for extra potential clusters (no nonzero columns remain) + # TODO: note that if a cluster is "removed" on a later iteration we will miss this. + # probably doesn't matter, just means it is occasionally possible that the V + # that is returned could sometimes have a few nonzero columns. + V_hat = np.concatenate((V_hat, np.zeros_like(V_hat)), -1) + K_max = V_hat.shape[1] + + # drop columns that are zeros (i.e. unused clusters) before returning + empty = V_hat.sum(0) == 0 + return V_hat[:,~empty], loss diff --git a/model/domain_assignment/util.py b/model/domain_assignment/util.py new file mode 100644 index 0000000000000000000000000000000000000000..17945d25a9321dd7a5473ac1ea967d3f693172fc --- /dev/null +++ b/model/domain_assignment/util.py @@ -0,0 +1,132 @@ +""" +Utilties for converting to/from a dictionary representation of domain assignments. +""" + +import logging +import os +from itertools import product + +import numpy as np + +import warnings +from Bio.PDB.PDBExceptions import PDBConstructionWarning +warnings.simplefilter('ignore', PDBConstructionWarning) + +LOG = logging.getLogger(__name__) + + +def make_pair_labels(n_res, domain_dict, id_string=None, save_dir=None, non_aligned_residues=[]): + """n_res: number of residues in the non-trimmed sequence + + non_aligned_residues: these will be used to trim down from n_res + + domain_dict: eg. {'D1': [0,1,2,3], 'D2': [4,5,6]} + """ + pair_labels = np.zeros([n_res, n_res]) + for domain, res_ix in domain_dict.items(): + if domain == 'linker': + continue + coords_tuples = list(product(res_ix, res_ix)) + x_ix = [i[0] for i in coords_tuples] + y_ix = [i[1] for i in coords_tuples] + pair_labels[x_ix, y_ix] = 1 + if len(non_aligned_residues): + aligned_residues = [i for i in range(n_res) if i not in non_aligned_residues] + pair_labels = pair_labels[aligned_residues,:][:,aligned_residues] + if save_dir is not None: + save_path = os.path.join(save_dir, id_string) + np.savez_compressed(save_path, pair_labels) + + return pair_labels + + +def sort_domain_limits(limits, dom_names): + start_positions = [x[0] for x in limits] + end_positions = [x[1] for x in limits] + sorted_index = np.argsort(start_positions) + assert (sorted_index == np.argsort(end_positions)).all() + return np.array(limits)[sorted_index], list(np.array(dom_names)[sorted_index]) + + +def resolve_residue_in_multiple_domain(mapping, shared_res): + """ + This is a stupid slow recursive solution: but I think it only applies to one + case so going to leave it for now + """ + for one_shared in shared_res: + for domain, res in mapping.items(): + if one_shared in res: + mapping[domain].remove(one_shared) + return check_no_residue_in_multiple_domains(mapping) + + +def check_no_residue_in_multiple_domains(mapping, resolve_conflics=True): + # ensures no residue index is associated with more than one domain + for dom, res in mapping.items(): + for dom2, res2 in mapping.items(): + if dom == dom2: + continue + shared_res = set(res).intersection(set(res2)) + if len(shared_res): + print(f'Found {len(shared_res)} shared residues') + if resolve_conflics: + mapping = resolve_residue_in_multiple_domain(mapping, shared_res) + else: + raise ValueError("SAME RESIDUE NUMBER FOUND IN MULTIPLE DOMAINS") + return mapping + + +def make_domain_mapping_dict(row): + dom_limit_list = row.dom_bounds_pdb_ix.split('|') + dom_names = row.dom_names.split('|') + dom_limit_list = convert_limits_to_numbers(dom_limit_list) + dom_limit_array, dom_names = sort_domain_limits(dom_limit_list, dom_names) + mapping = {} + + for i, d_lims in enumerate(dom_limit_array): + dom_name = dom_names[i] + pdb_start, pdb_end = d_lims + if dom_name not in mapping: + mapping[dom_name] = [] + mapping[dom_name] += list(range(pdb_start, pdb_end)) + check_no_residue_in_multiple_domains(mapping) + return mapping + + +def convert_limits_to_numbers(dom_limit_list): + processed_dom_limit_list = [] + for lim in dom_limit_list: + dash_idx = [i for i, char in enumerate(lim) if char == '-'] + if len(dash_idx) == 1: + start_index = int(lim.split('-')[0]) -1 + end_index = int(lim.split('-')[1]) + else: + raise ValueError('Invalid format for domain limits', str(dom_limit_list)) + processed_dom_limit_list.append((start_index, end_index)) + return processed_dom_limit_list + + +def convert_domain_dict_strings(domain_dict): + """ + Converts the domain dictionary into domain_name string and domain_bounds string + eg. domain names D1|D2|D1 + eg. domain bounds 0-100|100-200|200-300 + """ + domain_names = [] + domain_bounds = [] + for k,v in domain_dict.items(): + if k=='linker': + continue + residues = sorted(v) + for i, res in enumerate(residues): + if i==0: + start = res + elif residues[i-1] != res - 1: + domain_bounds.append(f'{start}-{residues[i-1]}') + domain_names.append(k) + start = res + if i == len(residues)-1: + domain_bounds.append(f'{start}-{res}') + domain_names.append(k) + + return '|'.join(domain_names), '|'.join(domain_bounds) diff --git a/model/domain_chop.py b/model/domain_chop.py new file mode 100644 index 0000000000000000000000000000000000000000..c9092ec4e0b9d5c3a35de0c20fb4644d4d415f03 --- /dev/null +++ b/model/domain_chop.py @@ -0,0 +1,340 @@ +"""Domain predictor classes. +""" +import os +import glob +import numpy as np +import pandas as pd +import torch +from torch import nn +from model.domain_assignment.util import make_pair_labels, make_domain_mapping_dict + +import logging +LOG = logging.getLogger(__name__) + + +def get_checkpoint_epoch(checkpoint_file): + return int(os.path.splitext(checkpoint_file)[0].split(".")[-1]) + + +class PairwiseDomainPredictor(nn.Module): + + """Wrapper for a pairwise domain co-membership predictor, adding in domain prediction post-processing.""" + + def __init__( + self, + model, + domain_caller, + device, + loss="bce", + x_has_padding_mask=True, + mask_padding=True, + n_checkpoints_to_average=1, + checkpoint_dir=None, + load_checkpoint_if_exists=False, + save_val_best=True, + max_recycles=0, + post_process_domains=True, + min_ss_components=2, + min_domain_length=30, + remove_disordered_domain_threshold=0, + trim_each_domain=True, + dist_transform_type="min_replace_inverse", + distance_denominator=10, + ): + super().__init__() + self._train_model = model # we want to keep this hanging around so that optimizer references dont break + self.model = self._train_model + self.domain_caller = domain_caller + self.device = device + self.x_has_padding_mask = x_has_padding_mask + self.mask_padding = mask_padding # if True use padding mask to mask loss + self.n_checkpoints_to_average = n_checkpoints_to_average + self.checkpoint_dir = checkpoint_dir + self._epoch = 0 + self.save_val_best = save_val_best + self.best_val_metrics = {} + self.max_recycles = max_recycles + self.post_process_domains = post_process_domains + self.remove_disordered_domain_threshold = remove_disordered_domain_threshold + self.trim_each_domain = trim_each_domain + self.min_domain_length = min_domain_length + self.min_ss_components = min_ss_components + self.dist_transform_type = dist_transform_type + self.distance_denominator = distance_denominator + if load_checkpoint_if_exists: + checkpoint_files = sorted( + glob.glob(os.path.join(self.checkpoint_dir, "weights*")), + key=get_checkpoint_epoch, + reverse=True, + ) + if len(checkpoint_files) > 0: + self._epoch = get_checkpoint_epoch(checkpoint_files[0]) + LOG.info(f"Loading saved checkpoint(s) ending at epoch {self._epoch}") + self.load_checkpoints(average=True) + self.load_checkpoints() + else: + LOG.info("No checkpoints found to load") + + if loss == "bce": + self.loss_function = nn.BCELoss(reduction="none") + elif loss == "mse": + self.loss_function = nn.MSELoss(reduction="none") + + def load_checkpoints(self, average=False, old_style=False): + if self.n_checkpoints_to_average == 1: + weights_file = os.path.join(self.checkpoint_dir, "weights.pt") + else: + # for e.g. resetting training weights for next training epoch after testing with avg + LOG.info(f"Loading last checkpoint (epoch {self._epoch})") + weights_file = os.path.join(self.checkpoint_dir, f"weights.{self._epoch}.pt") + LOG.info(f"Loading weights from: {weights_file}") + state_dict = torch.load(weights_file, map_location=self.device) + if old_style: + self.load_state_dict(state_dict, strict=False) + else: + self.model.load_state_dict(state_dict) + + def predict_pairwise(self, x): + x = x.to(self.device) + if np.isnan(x.cpu().numpy()).any(): + raise Exception('NAN values in data') + y_pred = self.model(x).squeeze(1) # b, L, L + assert y_pred.ndim == 3 + return y_pred + + def get_mask(self, x): + """Binary mask 1 for observed, 0 for padding.""" + x = x.to(self.device) + if self.x_has_padding_mask: + mask = 1 - x[:, -1] # b, L, L + else: + mask = None + return mask + + def epoch_start(self): + self.model = self._train_model + self.model.train() + self._epoch += 1 + + def test_begin(self): + if self.n_checkpoints_to_average > 1: + torch.save(self.model.state_dict(), os.path.join(self.checkpoint_dir, f'weights.{self._epoch}.pt')) + start_idx = self._epoch - self.n_checkpoints_to_average + if start_idx >= 2: + os.remove(os.path.join(self.checkpoint_dir, f"weights.{start_idx-1}.pt")) + else: + torch.save(self.model.state_dict(), os.path.join(self.checkpoint_dir, 'weights.pt')) + + if self.n_checkpoints_to_average > 1: + # self.model.to("cpu") # free up gpu memory for average model + self.load_checkpoints(average=True) + + self.model.eval() + + def forward(self, x, y, batch_average=True): + """A training step.""" + x, y = x.to(self.device), y.to(self.device) + y_pred = self.predict_pairwise(x) + mask = self.get_mask(x) + return self.compute_loss(y_pred, y, mask=mask) + + def compute_loss(self, y_pred, y, mask=None, batch_average=True): + y_pred, y = y_pred.to(self.device), y.to(self.device) + if mask is None or not self.mask_padding: + mask = torch.ones_like(y) + # mask is b, L, L. To normalise correctly, we need to divide by number of observations + loss = (self.loss_function(y_pred, y)*mask).sum((-1,-2)) / mask.sum((-1,-2)) + + # metrics characterising inputs: how many residues, how many with domain assignments. + labelled_residues = ((y*mask).sum(-1) > 0).sum(-1) # b + non_padding_residues = (mask.sum(-1) > 0).sum(-1) # b + labelled_frac = labelled_residues / non_padding_residues + metrics = { + "labelled_residues": labelled_residues.detach().cpu().numpy(), + "residues": non_padding_residues.detach().cpu().numpy(), + "labelled_frac": labelled_frac.detach().cpu().numpy(), + "loss": loss.detach().cpu().numpy(), + } + if batch_average: + loss = loss.mean(0) + metrics = {k: np.mean(v) for k, v in metrics.items()} + + return loss, metrics + + def domains_from_pairwise(self, y_pred): + assert y_pred.ndim == 3 + domain_preds = [] + confidence_list = [] + for pred_single in y_pred.cpu().numpy(): + single_domains, confidence = self.domain_caller(pred_single) + domain_preds.append(single_domains) + confidence_list.append(confidence) + return domain_preds, confidence_list + + def distance_transform(self, x): + dist_chan = x[0, 0] + # Find the minimum non-zero value in the channel + min_nonzero = dist_chan[dist_chan > 0].min() + # Replace zero values in the channel with the minimum non-zero value + dist_chan[dist_chan == 0] = min_nonzero + if self.dist_transform_type == "min_replace_inverse": + # replace zero values and then invert. + dist_chan = dist_chan ** (-1) + x[0, 0] = dist_chan + return x + + elif self.dist_transform_type == "unidoc_exponent": # replace zero values in pae / distance + spread = self.distance_denominator + dist_chan = (1 + np.exp((dist_chan - 8) / spread)) ** -1 + x[0,0] = dist_chan + return x + + @torch.no_grad() + def predict(self, x, return_pairwise=True): + x = self.distance_transform(x) + if self.max_recycles > 0: + for i in range(self.max_recycles): + # add recycling channels + n_res = x.shape[-1] + recycle_channels = torch.zeros(1, 2, n_res, n_res) + # Concatenate the original tensor and the zeros tensor along the second dimension + x = torch.cat((x, recycle_channels), dim=1) + x = self.recycle_predict(x) + y_pred = self.predict_pairwise(x) + domain_dicts, confidence = self.domains_from_pairwise(y_pred) + if self.post_process_domains: + domain_dicts = self.post_process(domain_dicts, x) # todo move this to domains from pairwise function + if return_pairwise: + return y_pred, domain_dicts, confidence + else: + return domain_dicts, confidence + + @torch.no_grad() + def recycle_predict(self, x): + x = x.to(self.device) + y_pred = self.predict_pairwise(x) + domain_dicts, confidence = self.domains_from_pairwise(y_pred) + y_pred_from_domains = np.array( + [make_pair_labels(n_res=x.shape[-1], domain_dict=d_dict) for d_dict in domain_dicts]) + y_pred_from_domains = torch.tensor(y_pred_from_domains).to(self.device) + if self.x_has_padding_mask: + x[:, -2, :, :] = y_pred # assumes that last dimension is padding mask + x[:, -3, :, :] = y_pred_from_domains + else: + x[:, -1, :, :] = y_pred + x[:, -2, :, :] = y_pred_from_domains + return x + + + def post_process(self, domain_dicts, x_batch): + new_domain_dicts = [] + for domain_dict, x in zip(domain_dicts, x_batch): + x = x.cpu().numpy() + domain_dict = {k: list(v) for k, v in domain_dict.items()} + helix, sheet = x[1], x[2] + diag_helix = np.diagonal(helix) + diag_sheet = np.diagonal(sheet) + ss_residues = list(np.where(diag_helix > 0)[0]) + list(np.where(diag_sheet > 0)[0]) + + domain_dict = self.trim_disordered_boundaries(domain_dict, ss_residues) + + if self.remove_disordered_domain_threshold > 0: + domain_dict = self.remove_disordered_domains(domain_dict, ss_residues) + + if self.min_ss_components > 0: + domain_dict = self.remove_domains_with_few_ss_components(domain_dict, x) + + if self.min_domain_length > 0: + domain_dict = self.remove_domains_with_short_length(domain_dict) + new_domain_dicts.append(domain_dict) + return new_domain_dicts + + def trim_disordered_boundaries(self, domain_dict, ss_residues): + if not self.trim_each_domain: + start = min(ss_residues) + end = max(ss_residues) + for dname, res in domain_dict.items(): + if dname == "linker": + continue + if self.trim_each_domain: + domain_specific_ss = set(ss_residues).intersection(set(res)) + if len(domain_specific_ss) == 0: + continue + start = min(domain_specific_ss) + end = max(domain_specific_ss) + domain_dict["linker"] += [r for r in res if r < start or r > end] + domain_dict[dname] = [r for r in res if r >= start and r <= end] + return domain_dict + + def remove_disordered_domains(self, domain_dict, ss_residues): + new_domain_dict = {} + for dname, res in domain_dict.items(): + if dname == "linker": + continue + if len(res) == 0: + continue + if len(set(res).intersection(set(ss_residues))) / len(res) < self.remove_disordered_domain_threshold: + domain_dict["linker"] += res + else: + new_domain_dict[dname] = res + new_domain_dict["linker"] = domain_dict["linker"] + return new_domain_dict + + + def remove_domains_with_few_ss_components(self, domain_dict, x): + """ + Remove domains where number of ss components is less than minimum + eg if self.min_ss_components=2 domains made of only a single helix or sheet are removed + achieve this by counting the number of unique string hashes in domain rows of x + """ + new_domain_dict = {} + for dname, res in domain_dict.items(): + if dname == "linker": + continue + res = sorted(res) + helix = x[1][res, :][:, res] + strand = x[2][res, :][:, res] + helix = helix[np.any(helix, axis=1)] + strand = strand[np.any(strand, axis=1)] + # residues in the same secondary structure component have the same representation in the helix or strand matrix + n_helix = len(set(["".join([str(int(i)) for i in row]) for row in helix])) + n_sheet = len(set(["".join([str(int(i)) for i in row]) for row in strand])) + if len(res) == 0: + continue + if n_helix + n_sheet < self.min_ss_components: + domain_dict["linker"] += res + else: + new_domain_dict[dname] = res + new_domain_dict["linker"] = domain_dict["linker"] + return new_domain_dict + + def remove_domains_with_short_length(self, domain_dict): + """ + Remove domains where length is less than minimum + """ + new_domain_dict = {} + for dname, res in domain_dict.items(): + if dname == "linker": + continue + + + if len(res) < self.min_domain_length: + domain_dict["linker"] += res + else: + new_domain_dict[dname] = res + new_domain_dict["linker"] = domain_dict["linker"] + return new_domain_dict + + +class CSVDomainPredictor: + def __init__(self, csv_predictions): + self.csv_filepath = csv_predictions + self.predictions = pd.read_csv(csv_predictions) + + def predict(self, x): + # x should be a chain_id + one_pred = self.predictions[self.predictions.chain_id == x[0][:5]] + if len(one_pred) == 0: + return None + domain_dicts = [make_domain_mapping_dict(one_pred.iloc[0])] + return None, domain_dicts diff --git a/model/errors.py b/model/errors.py new file mode 100644 index 0000000000000000000000000000000000000000..3c2ddabe5ee0650152e3e968a17cf6970c8691b3 --- /dev/null +++ b/model/errors.py @@ -0,0 +1,11 @@ +class BaseError(Exception): + """ + Error class for all local exceptions in this code base + """ + pass + +class PredictionResultExistsError(BaseError): + pass + +class FileExistsError(BaseError): + pass \ No newline at end of file diff --git a/model/factories.py b/model/factories.py new file mode 100644 index 0000000000000000000000000000000000000000..4f97f88fc593fa5b90442ff978325e29dd0387a1 --- /dev/null +++ b/model/factories.py @@ -0,0 +1,72 @@ +"""There are three configurable things: predictors/models, data, training/evaluation. + +Only first two require factories. +""" +import os + +import pandas as pd + +import logging + +from model.domain_chop import PairwiseDomainPredictor +from model.models.rosetta import trRosettaNetwork +from model.domain_assignment.assigners import SparseLowRank +from model.utils import common as common_utils + + +LOG = logging.getLogger(__name__) + + +def get_assigner(config): + assigner_type = config["type"] + if assigner_type == "sparse_lowrank": + assigner = SparseLowRank(**config["kwargs"]) + else: + return ValueError() + return assigner + + +def get_model(config): + model_type = config["type"] + if model_type == "trrosetta": + model = trRosettaNetwork(**config["kwargs"]) + else: + return ValueError() + return model + + +def pairwise_predictor( + learner_config, + force_cpu=False, + output_dir=None, + use_mps_if_available=False, +): + model = get_model(learner_config["model"]) + assigner = get_assigner(learner_config["assignment"]) + device = common_utils.get_torch_device( + force_cpu=force_cpu, + use_mps_if_available=use_mps_if_available, + ) + model.to(device) + kwargs = {k: v for k, v in learner_config.items() if k not in ["model", + "assignment", + "save_every_epoch", + "uncertainty_model" + ]} + LOG.info(f"Learner kwargs: {kwargs}") + return PairwiseDomainPredictor(model, assigner, device, checkpoint_dir=output_dir, **kwargs) + + +def get_test_ids(label_path, feature_path, csv_path=None): + ids = [id.split('.')[0] for id in set(os.listdir(label_path)).intersection(set(os.listdir(feature_path)))] + if csv_path is not None: + df = pd.read_csv(csv_path) + ids = sorted(list(set(ids).intersection(set(df.chain_id)))) + return ids + + +def filter_plddt(df_path, ids, threshold=90): + df = pd.read_csv(df_path) + df = df[df.plddt > threshold] + ids = [i for i in ids if i in df.casp_id.values] + return ids diff --git a/model/featurisers.py b/model/featurisers.py new file mode 100644 index 0000000000000000000000000000000000000000..da798e77f7f0b44ebbbab81168750fcc6e09dd7f --- /dev/null +++ b/model/featurisers.py @@ -0,0 +1,159 @@ +import logging +import os +from typing import List + +import Bio.PDB +import numpy as np +import torch +from scipy.spatial import distance_matrix + +from model import constants +from model.constants import _3to1 +from model.utils.cif2pdb import cif2pdb +from model.utils.secondary_structure import renum_pdb_file, \ + calculate_ss, make_ss_matrix + +LOG = logging.getLogger(__name__) + + +def get_model_structure(structure_path) -> Bio.PDB.Structure: + """ + Returns the Bio.PDB.Structure object for a given PDB or MMCIF file + """ + structure_id = os.path.split(structure_path)[-1].split('.')[0] + if structure_path.endswith('.pdb'): + structure = Bio.PDB.PDBParser().get_structure(structure_id, structure_path) + elif structure_path.endswith('.cif'): + structure = Bio.PDB.MMCIFParser().get_structure(structure_id, structure_path) + else: + raise ValueError(f'Unrecognized file extension: {structure_path}') + model = structure[0] + return model + + +class Residue: + def __init__(self, index: int, res_label: str, aa: str): + self.index = int(index) + self.res_label = str(res_label) + self.aa = str(aa) + +def get_model_structure_residues(structure_model: Bio.PDB.Structure, chain='A') -> List[Residue]: + """ + Returns a list of residues from a given PDB or MMCIF structure + """ + residues = [] + res_index = 1 + for biores in structure_model[chain].child_list: + res_num = biores.id[1] + res_ins = biores.id[2] + res_label = str(res_num) + if res_ins != ' ': + res_label += str(res_ins) + + aa3 = biores.get_resname() + if aa3 not in _3to1: + continue + + aa = _3to1[aa3] + res = Residue(res_index, res_label, aa) + residues.append(res) + + # increment the residue index after we have filtered out non-standard amino acids + res_index += 1 + + return residues + + +def inference_time_create_features(pdb_path, feature_config, chain="A", *, + model_structure: Bio.PDB.Structure=None, + renumber_pdbs=True, stride_path=constants.STRIDE_EXE, + ): + if pdb_path.endswith(".cif"): + pdb_path = cif2pdb(pdb_path) + + if not model_structure: + model_structure = get_model_structure(pdb_path) + + dist_matrix = get_distance(model_structure, chain=chain) + if renumber_pdbs: + output_pdb_path = pdb_path.replace('.pdb', '_renum.pdb').replace('.cif', '_renum.cif') + renum_pdb_file(pdb_path, output_pdb_path) + else: + output_pdb_path = pdb_path + ss_filepath = pdb_path + '_ss.txt' + calculate_ss(output_pdb_path, chain, stride_path, ssfile=ss_filepath) + helix, strand = make_ss_matrix(ss_filepath, nres=dist_matrix.shape[-1]) + if feature_config['ss_bounds']: + end_res_val = -1 if feature_config['negative_ss_end'] else 1 + helix_boundaries = make_boundary_matrix(helix, end_res_val=end_res_val) + strand_boundaries = make_boundary_matrix(strand, end_res_val=end_res_val) + if renumber_pdbs: + os.remove(output_pdb_path) + os.remove(ss_filepath) + LOG.info(f"Distance matrix shape: {dist_matrix.shape}, SS matrix shape: {helix.shape}") + if feature_config['ss_bounds']: + if feature_config['same_channel_boundaries_and_ss']: + helix_boundaries[helix == 1] = 1 + strand_boundaries[strand == 1] = 1 + stacked_features = np.stack((dist_matrix, helix_boundaries, strand_boundaries), axis=0) + else: + stacked_features = np.stack((dist_matrix, helix, strand, helix_boundaries, strand_boundaries), axis=0) + else: + stacked_features = np.stack((dist_matrix, helix, strand), axis=0) + stacked_features = stacked_features[None] # add batch dimension + return torch.Tensor(stacked_features) + + +def distance_matrix(x): + """Compute the distance matrix. + + Returns the matrix of all pair-wise distances. + + Parameters + ---------- + x : (M, K) array_like + Matrix of M vectors in K dimensions. + p : float, 1 <= p <= infinity + Which Minkowski p-norm to use. + Returns + ------- + result : (M, M) ndarray + Matrix containing the distance from every vector in `x` to every vector + in `x`. + """ + x1 = x[:, np.newaxis, :] # Expand x to 3D for broadcasting, shape (M, 1, K) + x2 = x[np.newaxis, :, :] # Expand x to 3D for broadcasting, shape (1, M, K) + + distance_matrix = np.sum(np.abs(x2 - x1) ** 2, axis=-1) ** 0.5 + + return distance_matrix.astype(np.float16) + + +def get_distance(structure_model: Bio.PDB.Structure, chain='A'): + alpha_coords = np.array([residue['CA'].get_coord() for residue in \ + structure_model[chain].get_residues() if Bio.PDB.is_aa(residue) and \ + 'CA' in residue and residue.get_resname() in _3to1], dtype=np.float16) + x = distance_matrix(alpha_coords) + return x + + +def make_boundary_matrix(ss, end_res_val=1): + """ + makes a matrix where the boundary residues + of the sec struct component are 1 + """ + ss_lines = np.zeros_like(ss) + diag = np.diag(ss) + if max(diag) == 0: + return ss_lines + padded_diag = np.zeros(len(diag) + 2) + padded_diag[1:-1] = diag + diff_before = diag - padded_diag[:-2] + diff_after = diag - padded_diag[2:] + start_res = np.where(diff_before == 1)[0] + end_res = np.where(diff_after == 1)[0] + ss_lines[start_res, :] = 1 + ss_lines[:, start_res] = 1 + ss_lines[end_res, :] = end_res_val + ss_lines[:, end_res] = end_res_val + return ss_lines diff --git a/model/loggers.py b/model/loggers.py new file mode 100644 index 0000000000000000000000000000000000000000..16c5040c59b33eeff30af02cff1b7c5079c08246 --- /dev/null +++ b/model/loggers.py @@ -0,0 +1,130 @@ +import glob +import os + +import logging +LOG = logging.getLogger(__name__) + +def get_versioned_dir(output_dir, version=None, resume=False): + """version gets dir for specific version, resume gets dir for last version.""" + if version is None: + current_versions = glob.glob(os.path.join(output_dir, "version*")) + if current_versions: + last_version = max([int(os.path.basename(v).split("_")[1]) for v in current_versions]) + version = last_version if resume else last_version + 1 + else: + assert not resume, f"Passed resume True but no matching directories in {output_dir}" + version = 1 + + version_dir = os.path.join(output_dir, f"version_{version}") + return version_dir, version + + +def log_epoch_metrics( + epoch, + metrics, + output_file, + extra_keys=None, + start_epoch=0, + new_file=False +): + """ + New file gets created if epoch == 1 + + We are going for a hierarchical structure /experiment_group/model_name/train_metrics.csv etc + because this works best with tensorboard and avoids file clutter in a single + experiment_group directory + + tensorboard refs: + https://pytorch.org/docs/stable/tensorboard.html + https://pytorch.org/tutorials/recipes/recipes/tensorboard_with_pytorch.html + """ + # output_filename = (model_name + f"_{msa_name}" + f"_vae" + + # ("_posembed{args.pos_embed_dim}" if args.embed_pos else "")) + + metrics.pop("epoch", None) + metric_names = list(metrics.keys()) + extra_keys = extra_keys or [] + assert all([m not in metric_names for m in extra_keys]), f"{metric_names} {extra_keys}" + metric_names += list(extra_keys) + + if new_file: # c.f. training/core epoch 0 is for validation. + with open(output_file, "w") as csvf: + csvf.write(",".join(["epoch"] + metric_names) + "\n") + + with open(output_file, "a") as csvf: + csvf.write(",".join([str(epoch + start_epoch)] + [str(metrics.get(m, "")) for m in metric_names])+"\n") + + +class StdOutLogger: + + def __init__(self, log_freq, start_epoch=0): + self.start_epoch = start_epoch + self.log_freq = log_freq + + def log(self, epoch, metrics, batch=None): + if self.log_freq is not None and epoch % self.log_freq == 0: + if batch is None: + header = f"Epoch {epoch + self.start_epoch}: " + else: + header = f"[{epoch:d}, {batch:5d}]: " + + train_metric_components = [f"{m}: {v:.3f} " for m, v in metrics.items() if not m.startswith("val_")] + if train_metric_components: + LOG.info( + header + + " ".join(train_metric_components), + ) + val_metric_components = [f"{m}: {v:.3f} " for m, v in metrics.items() if m.startswith("val_")] + if val_metric_components: + LOG.info( + " ".join(val_metric_components), + ) + if batch is None: + LOG.info("--------------------------------------\n") + + +class CSVLogger: + def __init__(self, output_dir, start_epoch=0): + self.output_dir = output_dir + self.start_epoch = start_epoch + self.val_keys = None + self.filename = f"train_log.{'' if start_epoch == 0 else (str(start_epoch) + '.')}csv" + self.logged = 0 + + @property + def filepath(self): + return str(os.path.join(self.output_dir, self.filename)) + + def log(self, epoch, metrics, batch=None): + metrics["batch"] = batch + + if epoch == 0: + self.val_keys = metrics.keys() + elif self.output_dir is not None and epoch > 0: + # LOG.info([k for k in metrics.keys() if k not in self._prev_keys]) + extra_keys = [k for k in self.val_keys if k not in metrics and k != "epoch"] + os.makedirs(self.output_dir, exist_ok=True) + log_epoch_metrics( + epoch, + metrics, + self.filepath, + extra_keys=extra_keys, + start_epoch=self.start_epoch, + new_file=self.logged == 0 + ) + self.logged += 1 + # self._prev_keys = metrics.keys() + + +class LoggerContainer: + + def __init__(self, loggers, start_epoch=0): + self.train_log = [] + self.loggers = loggers + self.start_epoch = start_epoch + + def log(self, epoch, metrics, batch=None): + for logger in self.loggers: + logger.log(epoch, metrics, batch=batch) + metrics["epoch"] = epoch + self.start_epoch + self.train_log.append(metrics) diff --git a/model/models/results.py b/model/models/results.py new file mode 100644 index 0000000000000000000000000000000000000000..9e911a1e5465d28e31b8394365fb5da10923a4b4 --- /dev/null +++ b/model/models/results.py @@ -0,0 +1,23 @@ +from pathlib import Path +from typing import Optional +from pydantic import BaseModel, validator + +def chain_id_from_pdb_path(pdb_path: Path): + return pdb_path.stem + +class PredictionResult(BaseModel): + + pdb_path: Path + confidence: float + chain_id: Optional[str] = None + sequence_md5: str + ndom: int + nres: int + chopping: Optional[str] = None + time_sec: Optional[float] = None + + @validator('chain_id', always=True, pre=True, allow_reuse=True) + def set_chain_id(cls, v, values): + if v is None: + return chain_id_from_pdb_path(values['pdb_path']) + return v diff --git a/model/models/rosetta.py b/model/models/rosetta.py new file mode 100644 index 0000000000000000000000000000000000000000..340667fd3f5b3e1c9e36446a8a2ee6015627d639 --- /dev/null +++ b/model/models/rosetta.py @@ -0,0 +1,68 @@ +from torch import nn, transpose + + +def elu(): + return nn.ELU(inplace=True) + + +def instance_norm(filters, eps=1e-6, **kwargs): + return nn.InstanceNorm2d(filters, affine=True, eps=eps, **kwargs) + + +def conv2d(in_chan, out_chan, kernel_size, dilation=1, **kwargs): + padding = dilation * (kernel_size - 1) // 2 + return nn.Conv2d(in_chan, out_chan, kernel_size, padding=padding, dilation=dilation, **kwargs) + + +class trRosettaNetwork(nn.Module): + def __init__(self, filters=64, kernel=3, num_layers=61, in_channels=3, symmetrise_output=False, dropout=0.15): + super().__init__() + self.filters = filters + self.kernel = kernel + self.num_layers = num_layers + self.in_channels = in_channels + self.symmetrise_output = symmetrise_output + self.first_block = nn.Sequential( + conv2d(self.in_channels, filters, 1), + instance_norm(filters), + elu() + ) + self.output_layer = nn.Sequential( + conv2d(filters, 1, kernel, dilation=1), + nn.Sigmoid()) + + + # stack of residual blocks with dilations + cycle_dilations = [1, 2, 4, 8, 16] + dilations = [cycle_dilations[i % len(cycle_dilations)] for i in range(num_layers)] + if dropout > 0: + self.layers = nn.ModuleList([nn.Sequential( + conv2d(filters, filters, kernel, dilation=dilation), + instance_norm(filters), + elu(), + nn.Dropout(p=dropout), + conv2d(filters, filters, kernel, dilation=dilation), + instance_norm(filters) + ) for dilation in dilations]) + else: + self.layers = nn.ModuleList([nn.Sequential( + conv2d(filters, filters, kernel, dilation=dilation), + instance_norm(filters), + elu(), + conv2d(filters, filters, kernel, dilation=dilation), + instance_norm(filters) + ) for dilation in dilations]) + + self.activate = elu() + + + def forward(self, x): + x = self.first_block(x) + + for layer in self.layers: + x = self.activate(x + layer(x)) + y_hat = self.output_layer(x) + if self.symmetrise_output: + return (y_hat + transpose(y_hat, -1, -2)) * 0.5 + else: + return y_hat diff --git a/model/prediction_result_file.py b/model/prediction_result_file.py new file mode 100644 index 0000000000000000000000000000000000000000..c75d0a8e6fffd5c48a81a90d478db47231b0156c --- /dev/null +++ b/model/prediction_result_file.py @@ -0,0 +1,161 @@ +""" +Class that represents a file with the prediction results + +* allows prediction results to be appended to the file in chunks +* allows results to be skipped if they already exist in the file +""" + +import csv +from typing import List, OrderedDict +from pathlib import Path +import logging + +from .models.results import PredictionResult +from .errors import PredictionResultExistsError, FileExistsError + +LOG = logging.getLogger(__name__) + +class PredictionResultsFileIter: + def __init__(self, results_file_class): + self._results = results_file_class.get_results() + self._results_size = len(self._results) + self._current_index = 0 + + def __iter__(self): + return self + + def __next__(self): + if self._current_index < self._results_size: + member = self._results[self._current_index] + self._current_index += 1 + return member + raise StopIteration + + +class PredictionResultsFile: + """ + Writes prediction results to a file in chunks + """ + + COLNAMES = ['chain_id', 'sequence_md5', 'nres', 'ndom', 'chopping', 'confidence', 'time_sec'] + + def __init__(self, + csv_path: Path, *, + chunk_size: int=20, + write_header: bool=True, + allow_append: bool=False, + allow_skip: bool=False): + + self.csv_path = Path(str(csv_path)).absolute() + self.chunk_size = chunk_size + self.write_header = write_header + self.allow_append = allow_append + self.allow_skip = allow_skip + self._results_by_id: OrderedDict[PredictionResult] = OrderedDict() + self._unflushed_results_by_id: OrderedDict[PredictionResult] = OrderedDict() + self._written_header = False + + self._init() + + def _init(self): + if self.csv_path.exists(): + msg = f"file '{self.csv_path}' already exists (allow_append={self.allow_append}))" + LOG.warning(msg) + if not self.allow_append: + raise FileExistsError(msg) + self._read_results() + + def _read_results(self): + """ + Read the results from the file + """ + with self.csv_path.open('r') as fp: + reader = self.get_csv_reader(fp) + for row in reader: + if row['chain_id'] == 'chain_id': + continue + pdb_path = Path(row['chain_id'] + '.pdb') + result = PredictionResult(pdb_path=pdb_path, **row) + self._results_by_id[result.chain_id] = result + + def get_flushed_results(self): + self.flush() + return self._results_by_id.values() + + def get_csv_reader(self, fp): + return csv.DictReader(fp, delimiter='\t', fieldnames=self.COLNAMES) + + def get_csv_writer(self, fp): + return csv.DictWriter(fp, delimiter='\t', fieldnames=self.COLNAMES) + + def write_csv_result(self, csv_writer, res: PredictionResult): + """ + Render PredictionResult result to file pointer + """ + row = { + 'chain_id': res.chain_id, + 'sequence_md5': res.sequence_md5, + 'nres': res.nres, + 'ndom': res.ndom, + 'chopping': res.chopping if res.chopping is not None else 'NULL', + 'confidence': f'{res.confidence:.3g}' if res.confidence is not None else 'NULL', + 'time_sec': f'{res.time_sec}' if res.time_sec is not None else 'NULL', + } + csv_writer.writerow(row) + + + def flush(self): + """ + Write unflushed results to the output file + """ + if len(self._unflushed_results_by_id) == 0: + return + + with self.csv_path.open('a') as fp: + + csv_writer = self.get_csv_writer(fp) + + if self.write_header and not self._written_header and len(self._results_by_id) == 0: + csv_writer.writeheader() + self._written_header = True + + for result in self._unflushed_results_by_id.values(): + self.write_csv_result(csv_writer, result) + + def has_result(self, result: PredictionResult): + """ + Check if the prediction result already exists in the file + """ + return self.has_result_for_chain_id(result.chain_id) + + def has_result_for_chain_id(self, chain_id: str): + """ + Check if the chain_id already exists in the file + """ + if chain_id in self._results_by_id: + return True + if chain_id in self._unflushed_results_by_id: + return True + + return False + + def add_result(self, result: PredictionResult): + """ + Add a result to the buffer (and possibly flush the buffer) + """ + if self.has_result(result): + msg = f"result '{result.chain_id}' already exists in file '{self.csv_path}'" + raise PredictionResultExistsError(msg) + + self._unflushed_results_by_id[result.chain_id] = result + if len(self._unflushed_results_by_id) >= self.chunk_size: + self.flush() + self._results_by_id.update(self._unflushed_results_by_id) + self._unflushed_results_by_id = OrderedDict() + + def add_results(self, results: List[PredictionResult]): + for res in results: + self.add_result(res) + + def __iter__(self): + return PredictionResultsFileIter(self) diff --git a/model/utils/__init__.py b/model/utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/model/utils/cif2pdb.py b/model/utils/cif2pdb.py new file mode 100644 index 0000000000000000000000000000000000000000..f8901dcdbf81d0cfbc4115e4691b6d8b76a10a4e --- /dev/null +++ b/model/utils/cif2pdb.py @@ -0,0 +1,137 @@ +# !/usr/bin/env python +""" +Script to convert mmCIF files to PDB format. +usage: python cif2pdb.py ciffile [pdbfile] +Requires python BioPython (`pip install biopython`). It should work with recent version of python 2 or 3. +@author Spencer Bliven +""" +import os +import sys +import argparse +import logging +from Bio.PDB.MMCIFParser import MMCIFParser +from Bio.PDB import PDBIO + +LOG = logging.getLogger(__name__) + + +def int_to_chain(i, base=62): + """ + int_to_chain(int,int) -> str + Converts a positive integer to a chain ID. Chain IDs include uppercase + characters, numbers, and optionally lowercase letters. + i = a positive integer to convert + base = the alphabet size to include. Typically 36 or 62. + """ + if i < 0: + raise ValueError("positive integers only") + if base < 0 or 62 < base: + raise ValueError("Invalid base") + + quot = int(i) // base + rem = i % base + if rem < 26: + letter = chr(ord("A") + rem) + elif rem < 36: + letter = str(rem - 26) + else: + letter = chr(ord("a") + rem - 36) + if quot == 0: + return letter + else: + return int_to_chain(quot - 1, base) + letter + + +class OutOfChainsError(Exception): + pass + + +def rename_chains(structure): + """Renames chains to be one-letter chains + + Existing one-letter chains will be kept. Multi-letter chains will be truncated + or renamed to the next available letter of the alphabet. + + If more than 62 chains are present in the structure, raises an OutOfChainsError + + Returns a map between new and old chain IDs, as well as modifying the input structure + """ + next_chain = 0 # + # single-letters stay the same + chainmap = {c.id: c.id for c in structure.get_chains() if len(c.id) == 1} + for o in structure.get_chains(): + if len(o.id) != 1: + if o.id[0] not in chainmap: + chainmap[o.id[0]] = o.id + o.id = o.id[0] + else: + c = int_to_chain(next_chain) + while c in chainmap: + next_chain += 1 + c = int_to_chain(next_chain) + if next_chain >= 62: + raise OutOfChainsError() + chainmap[c] = o.id + o.id = c + return chainmap + +def cif2pdb(ciffile): + pdbfile = ciffile + ".pdb" + strucid = os.path.split(ciffile)[-1].split(".")[0] + # Read file + parser = MMCIFParser() + structure = parser.get_structure(strucid, ciffile) + # rename long chains + try: + chainmap = rename_chains(structure) + except OutOfChainsError: + logging.error("Too many chains to represent in PDB format") + sys.exit(1) + + for new, old in chainmap.items(): + if new != old: + logging.info("Renaming chain {0} to {1}".format(old, new)) + + # Write PDB + io = PDBIO() + io.set_structure(structure) + io.save(pdbfile) + return pdbfile + + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Convert mmCIF to PDB format') + parser.add_argument("ciffile", help="mmCIF input file") + parser.add_argument("pdbfile", nargs="?", help="PDB output file. Default based on CIF filename") + parser.add_argument("-v", "--verbose", help="Long messages", + dest="verbose", default=False, action="store_true") + args = parser.parse_args() + + logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.DEBUG if args.verbose else logging.WARN) + + ciffile = args.ciffile + pdbfile = args.pdbfile or ciffile + ".pdb" + # Not sure why biopython needs this to read a cif file + strucid = ciffile[:4] if len(ciffile) > 4 else "1xxx" + + # Read file + parser = MMCIFParser() + structure = parser.get_structure(strucid, ciffile) + + # rename long chains + try: + chainmap = rename_chains(structure) + except OutOfChainsError: + logging.error("Too many chains to represent in PDB format") + sys.exit(1) + + if args.verbose: + for new, old in chainmap.items(): + if new != old: + logging.info("Renaming chain {0} to {1}".format(old, new)) + + # Write PDB + io = PDBIO() + io.set_structure(structure) + io.save(pdbfile) \ No newline at end of file diff --git a/model/utils/common.py b/model/utils/common.py new file mode 100644 index 0000000000000000000000000000000000000000..c1e4ef44e97d0e98ede6040faab770a30b734385 --- /dev/null +++ b/model/utils/common.py @@ -0,0 +1,49 @@ +import copy +import json +import subprocess +import torch + +import logging +LOG = logging.getLogger(__name__) + + +def load_json(jsonfile): + with open(jsonfile, "r") as jf: + res = json.load(jf) + return res + + +def save_config(config, filepath): + with open(filepath, "w") as outfile: + json.dump(config, outfile, indent=4) + + +def apply_diff(base_cfg, diff): + cfg = copy.deepcopy(base_cfg) + for k, v in diff.items(): + if isinstance(v, dict): + cfg[k] = apply_diff(base_cfg[k], v) + else: + cfg[k] = v + return cfg + + +def execute_bash_command(bash_command_string): + bash_return = subprocess.run(bash_command_string.split(), timeout=20) + return bash_return + + +def get_torch_device(force_cpu=False, use_mps_if_available=False): + try: + if torch.cuda.is_available() and not force_cpu: + device_string = "cuda" + elif use_mps_if_available and torch.backends.mps.is_available() and not force_cpu: + device_string = "mps" + else: + device_string = "cpu" + except Exception as exc: + LOG.error(f'Exception: {exc}') + device_string = "cpu" + LOG.info(f'Using device: {device_string}') + device = torch.device(device_string) + return device diff --git a/model/utils/domain_boundary_distance_score.py b/model/utils/domain_boundary_distance_score.py new file mode 100644 index 0000000000000000000000000000000000000000..14ebbe8d42e21fd2e5a05a2cb9c9d0ac4bf2f03a --- /dev/null +++ b/model/utils/domain_boundary_distance_score.py @@ -0,0 +1,79 @@ +""" +Domain boundary distance score, as defined in CASP 7 paper: +Assessment of predictions submitted for the CASP7 domain prediction category (2007) +https://onlinelibrary.wiley.com/doi/10.1002/prot.21675 + +Under our scoring scheme all predictions within 8 residues of the correct boundary +will score, but predictions that are closer to the correct domain boundary would score more. +All distances between the predicted and correct domain boundaries are calculated. If the +domain boundary has a linker, the whole linker is regarded as the domain boundary. +predictions are given one point for being within 1 residue of each correct boundary, +another point if they are within two residues, a further point if they are within three, +and so on up to eight residues. A prediction two residues away from the correct boundary +would therefore have 7 points. +The total score for each domain prediction is then calculated as the sum of all predicted +boundary scores divided by eight and the total number of domain boundaries. The number of +domain boundaries comes from either the target or the prediction, whichever is higher. In +this way over-prediction is penalized. + +""" +import numpy as np + + +def pred_domains_to_bounds(pred_domains, optimize_for_linkers=False): + """ + Converts domain dictionary to list of boundary residues + """ + pred_bounds = np.array([]) + for name, res in pred_domains.items(): + if name=='linker' or len(res) ==0: + continue + res = np.array(sorted(set(res))) + gaps = res[1:] - res[:-1] + if any(gaps > 1): + gaps_start_indexes = np.where(gaps > 1)[0] + pred_bounds = np.append(pred_bounds, res[gaps_start_indexes] + 1) + pred_bounds = np.append(pred_bounds, res[gaps_start_indexes + 1]) + pred_bounds = np.append(pred_bounds, [res[0], res[-1] +1]) + if optimize_for_linkers: + # given that linkers all count as boundaries probs possible + # to score more points by predicting boundaries in the middle of linker + # regions rather than the edges + raise NotImplementedError # todo + return np.array(sorted(list(set(pred_bounds.astype(int))))) + + +def get_true_boundary_res(domain_dict): + """ + In the case where there are multiple non-domain + residues between two domains, all of these NDRs + are counted as domain boundaries. This adjustment + is applied to the true boundaries but not the + predicted boundaries. + """ + bounds = pred_domains_to_bounds(domain_dict) + # c.f. get_boundary_res below + boundaries = { + "boundary_res": list(bounds), + "n_boundaries": len(list(bounds)) + } + boundaries["boundary_res"] += list(domain_dict["linker"]) + return boundaries + + +def boundary_distance_score(domains, boundaries): + pred_bounds = pred_domains_to_bounds(domains) + # distance score as specified in CASP7 paper {distance_to_true_bound:score} + dist_to_score = {0:8, 1:7, 2:6, 3:5, 4:4, 5:3, 6:2, 7:1} + score = 0 + # the final score is divided by the number of segments and the maximum score for each boundary + normalizing_term = 8 * max(len(pred_bounds), boundaries['n_boundaries']) + scores = [] + for b in pred_bounds: + distance = min(abs(boundaries['boundary_res'] - b)) + if distance < 8: + scores.append(dist_to_score[distance]) + # JW adjustment: additional boundaries that can't be mapped to a real bound + # should not be added to the un-normalized score + score = sum(sorted(scores)[-boundaries['n_boundaries']:]) + return score / normalizing_term diff --git a/model/utils/ndo_score.py b/model/utils/ndo_score.py new file mode 100644 index 0000000000000000000000000000000000000000..1a997db14e72a38f7d3d7d26971489e78c9f5d6a --- /dev/null +++ b/model/utils/ndo_score.py @@ -0,0 +1,94 @@ +from collections import defaultdict +import copy +import numpy as np + + +def check_unique_assignments(assn): + res_counts = defaultdict(int) # check for repeated residues + for res_ids in assn.values(): + for res_id in res_ids: + res_counts[res_id] += 1 + assert all([v == 1 for v in res_counts.values()]), f"Non-unique domain assignment {assn}" + + +def make_domain_dict(domain_str, n_res): + """ + Converts string representation of domain boundaries to a dictionary + Delimit separate domains with commas , and discontinuous domains + with underscores _. Residue ranges separated by hyphens -, e.g. 1-100,101-200_300-340 + :param domain_str: domain boundaries expressed zero-indexed sequential e.g. 1-100,101-200_300-340 + :param n_res: number of residues in the sequence + :return: + """ + domain_dict = {} + bounds = domain_str.split(',') + assigned_res = set() + for i, bound in enumerate(bounds): + if len(bound): + + for segment in bound.split('_'): + if '-' in segment: + start, end = segment.split('-') + else: + start = end = segment + segment_res = set(range(int(start), int(end) + 1)) + assert len(segment_res.intersection(assigned_res)) == 0, f"Overlapping domain assignments {domain_str}" + assigned_res.update(segment_res) + domain_dict[f"D{i + 1}"] = domain_dict.get(f"D{i + 1}", []) + list(segment_res) + domain_dict['linker'] = list(set(range(n_res)).difference(assigned_res)) + return domain_dict + + +def ndo_score(true: dict, pred: dict): + """ + Normalized Domain Overlap Score + Approximately corresponds to the fraction of residues that are assigned to the correct domain + (Tai, C.H., et al. Evaluation of domain prediction in CASP6. Proteins 2005;61:183-192.) + Full description of algorithm at https://ccrod.cancer.gov/confluence/display/CCRLEE/NDO] + https://ccrod.cancer.gov/confluence/display/CCRLEE/NDO + :param true: dict of domain assignments, with keys "linker" and "dX" where X is the domain number + :param pred: dict of domain assignments, with keys "linker" and "dX" where X is the domain number + example domain dictionary: {'linker':[0,1], 'D1':[2,3,4,8,9], 'D2':[5,6,7]} + """ + + # domains definitions must be mutually exclusive. + check_unique_assignments(true) + check_unique_assignments(pred) + # alternative data structure would be a list of res ids for linker, plus a list of lists of res ids for domains. + true = copy.deepcopy(true) + pred = copy.deepcopy(pred) + + n_dom_pred = len([k for k in pred.keys() if k != "linker"]) + n_dom_gt = len([k for k in true.keys() if k != "linker"]) + + # linkers are treated specially, so put them at row/col 0 to make this easy + + pred_linker = pred.pop("linker", []) + gt_linker = true.pop("linker", []) + + gt_res_ids = [gt_linker] + list(true.values()) + pred_res_ids = [pred_linker] + list(pred.values()) + + overlap = np.zeros((n_dom_pred + 1, n_dom_gt + 1)) + assert len(gt_res_ids) == overlap.shape[1] and len(pred_res_ids) == overlap.shape[0] + + for i, p_res in enumerate(pred_res_ids): + for j, gt_res in enumerate(gt_res_ids): + overlap[i, j] = len(set(p_res).intersection(set(gt_res))) + + # modified from v0: max overlap for a domain cannot be with a linker (hence max slices start at 1) + if overlap.shape[0] == 1 or overlap.shape[1] == 1: + # either the predictions or the true labels (or both) are all linker + print('NDO score undefined for case where all linker') + return 0 + row_scores = overlap[1:, 1:].max(axis=1) + row_scores -= (overlap[1:].sum(axis=1) - row_scores) + + col_scores = overlap[1:, 1:].max(axis=0) + col_scores -= (overlap[:, 1:].sum(axis=0) - col_scores) + + total_score = (row_scores.sum() + col_scores.sum()) / 2 + # count number of domain (non-linker) residues to normalize + max_score = sum([len(gt_res) for gt_res in gt_res_ids[1:]]) + + return total_score / max_score diff --git a/model/utils/pdb_reres.py b/model/utils/pdb_reres.py new file mode 100644 index 0000000000000000000000000000000000000000..ab6dcb346bb25ab596bf0b16b652410f5c25bf37 --- /dev/null +++ b/model/utils/pdb_reres.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright 2018 João Pedro Rodrigues +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Renumbers the residues of the PDB file starting from a given number (default 1). + +Usage: + python pdb_reres.py - + +Example: + python pdb_reres.py -10 1CTF.pdb # renumbers from 10 + python pdb_reres.py --1 1CTF.pdb # renumbers from -1 + +This program is part of the `pdb-tools` suite of utilities and should not be +distributed isolatedly. The `pdb-tools` were created to quickly manipulate PDB +files using the terminal, and can be used sequentially, with one tool streaming +data to another. They are based on old FORTRAN77 code that was taking too much +effort to maintain and compile. RIP. +""" + +import os +import sys + +import logging +LOG = logging.getLogger(__name__) + + +__author__ = "Joao Rodrigues" +__email__ = "j.p.g.l.m.rodrigues@gmail.com" + + +def check_input(args): + """Checks whether to read from stdin/file and validates user input/options. + """ + + # Defaults + option = 1 + fh = sys.stdin # file handle + + if not len(args): + # Reading from pipe with default option + if sys.stdin.isatty(): + sys.stderr.write(__doc__) + sys.exit(1) + + elif len(args) == 1: + # One of two options: option & Pipe OR file & default option + if args[0].startswith('-'): + option = args[0][1:] + if sys.stdin.isatty(): # ensure the PDB data is streamed in + emsg = 'ERROR!! No data to process!\n' + sys.stderr.write(emsg) + sys.stderr.write(__doc__) + sys.exit(1) + + else: + if not os.path.isfile(args[0]): + emsg = 'ERROR!! File not found or not readable: \'{}\'\n' + sys.stderr.write(emsg.format(args[0])) + sys.stderr.write(__doc__) + sys.exit(1) + + fh = open(args[0], 'r') + + elif len(args) == 2: + # Two options: option & File + if not args[0].startswith('-'): + emsg = 'ERROR! First argument is not an option: \'{}\'\n' + sys.stderr.write(emsg.format(args[0])) + sys.stderr.write(__doc__) + sys.exit(1) + + if not os.path.isfile(args[1]): + emsg = 'ERROR!! File not found or not readable: \'{}\'\n' + sys.stderr.write(emsg.format(args[1])) + sys.stderr.write(__doc__) + sys.exit(1) + + option = args[0][1:] + fh = open(args[1], 'r') + + else: # Whatever ... + sys.stderr.write(__doc__) + sys.exit(1) + + # Validate option + try: + option = int(option) + except ValueError: + emsg = 'ERROR!! You provided an invalid residue number: \'{}\'' + sys.stderr.write(emsg.format(option)) + sys.exit(1) + + return (fh, option) + + +def pad_line(line): + """Helper function to pad line to 80 characters in case it is shorter""" + size_of_line = len(line) + if size_of_line < 80: + padding = 80 - size_of_line + 1 + line = line.strip('\n') + ' ' * padding + '\n' + return line[:81] # 80 + newline character + + +def run(fhandle, starting_resid): + """ + Reset the residue number column to start from a specific number. + + This function is a generator. + + Parameters + ---------- + fhandle : a line-by-line iterator of the original PDB file. + + starting_resid : int + The starting residue number. + + Yields + ------ + str (line-by-line) + The modified (or not) PDB line. + """ + _pad_line = pad_line + prev_resid = None # tracks chain and resid + resid = starting_resid - 1 # account for first residue + records = ('ATOM', 'HETATM', 'TER', 'ANISOU') + for line in fhandle: + line = _pad_line(line) + if line.startswith('MODEL'): + resid = starting_resid - 1 # account for first residue + prev_resid = None # tracks chain and resid + yield line + + elif line.startswith(records): + line_resuid = line[17:27] + if line_resuid != prev_resid: + prev_resid = line_resuid + resid += 1 + if resid > 9999: + emsg = 'Cannot set residue number above 9999.\n' + sys.stderr.write(emsg) + sys.exit(1) + + yield line[:22] + str(resid).rjust(4) + line[26:] + + else: + yield line + + +renumber_residues = run + + +def main(): + # Check Input + pdbfh, starting_resid = check_input(sys.argv[1:]) + + # Do the job + new_pdb = run(pdbfh, starting_resid) + + # Output results + try: + _buffer = [] + _buffer_size = 5000 # write N lines at a time + for lineno, line in enumerate(new_pdb): + if not (lineno % _buffer_size): + sys.stdout.write(''.join(_buffer)) + _buffer = [] + _buffer.append(line) + + sys.stdout.write(''.join(_buffer)) + sys.stdout.flush() + except IOError: + # This is here to catch Broken Pipes + # for example to use 'head' or 'tail' without + # the error message showing up + pass + + # last line of the script + # Close file handle even if it is sys.stdin, no problem here. + pdbfh.close() + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/model/utils/pymol_3d_visuals.py b/model/utils/pymol_3d_visuals.py new file mode 100644 index 0000000000000000000000000000000000000000..56b2e82e1ccc95713cf763c43016623987a1e9b6 --- /dev/null +++ b/model/utils/pymol_3d_visuals.py @@ -0,0 +1,47 @@ +import os + +import subprocess + +import logging +LOG = logging.getLogger(__name__) + + +def generate_pymol_image(pdb_path, chopping, image_out_path, path_to_script, + pymol_executable): + """ + This function generates a pymol script which describes how one protein should be visualized + and saved as a png file. Script is then executed via subprocess. + Delimit separate domains with commas , and discontinuous domains + with underscores _. Residue ranges separated by hyphens -, e.g. 1-100,101-200_300-340 + :param pdb_path: string which is the path to a .pdb file + :param bounds: domain boundaries expressed in residue author numbers in the pdb file e.g. 1-100,101-200_300-340 + :param path_to_script: where the pymol bash script is saved + :param pymol_executable: path to pymol executable + :return: None + """ + script_text = f""" +load {pdb_path}, structure +hide everything +bg_color white +as cartoon, structure +set ray_opaque_background,1 +color white, structure\n""" + color_list = ['green', 'red', 'cyan', 'magenta', 'blue', 'orange', 'dirtyviolet', 'olive', + 'limon', 'salmon', 'deepteal', 'yellow', 'sand', 'purpleblue', 'black'] + bounds = chopping.split(',') + for i, bound in enumerate(bounds): + if len(bound): + domain_color = color_list[i % len(color_list)] + for segment in bound.split('_'): + start, end = segment.split('-') + script_text += f"color {domain_color}, structure and resi {start}-{end}\n" + script_text += "orient\n" + script_text += f"png {image_out_path}, ray=1, width=20cm, dpi=300,\n" + script_text += f"save {image_out_path.split('.')[0] + '.pse'}\n" + with open(path_to_script, 'w') as filehandle: + filehandle.write(script_text) + bash_command = f"{pymol_executable} -c {path_to_script}" + status = subprocess.run(bash_command.split(), timeout=120) + os.remove(path_to_script) + return status + diff --git a/model/utils/secondary_structure.py b/model/utils/secondary_structure.py new file mode 100644 index 0000000000000000000000000000000000000000..1bc927dd48bca32096056c34e557d72f486e0bf5 --- /dev/null +++ b/model/utils/secondary_structure.py @@ -0,0 +1,94 @@ +""" +Created by Jude Wells 2023-04-20 +Objective is to create a secondary structure matrix for each protein +1) renumbers the pdb file (1-indexed) +2) runs stride to get secondary structure file +3) parses the secondary structure file to create a matrix of secondary structure + +# should take around 3 minutes for 1000 structures +""" + +import os +import re +import subprocess +import sys + +import numpy as np + +from model.constants import REPO_ROOT +import logging +LOG = logging.getLogger(__name__) + + +def calculate_ss(pdbfile, chain, stride_path, ssfile='pdb_ss'): + assert os.path.exists(pdbfile) + with open(ssfile, 'w') as ssout_file: + args = [stride_path, pdbfile, '-r' + chain] + LOG.info(f"Running command: {' '.join(args)}") + try: + subprocess.run(args, + stdout=ssout_file, + stderr=subprocess.DEVNULL, + check=True) + except subprocess.CalledProcessError as e: + LOG.warning(f"Stride failed on {pdbfile}, creating empty file") + pass + + +def make_ss_matrix(ss_path, nres): + # create matrices for helix and strad residues where entry ij = 1 if i and j are in the same helix or strand + with open(ss_path) as f: + lines = f.readlines() + type_set = set() + helix = np.zeros([nres, nres], dtype=np.float32) + strand = np.zeros([nres, nres], dtype=np.float32) + for line in lines: + if line.startswith('LOC'): + start = int(re.sub(r'\D', '', line[22:28].strip())) + end = int(re.sub(r'\D', '', line[40:46].strip())) + type = line[5:17].strip() + type_set.add(type) + if type in ['AlphaHelix', '310Helix']: + helix[start-1:end, start-1:end] = 1 + elif type == 'Strand': + strand[start-1:end, start-1:end] = 1 + elif line.startswith('ASG'): + break + return helix, strand + + +def renum_pdb_file(pdb_path, output_pdb_path): + pdb_reres_path = REPO_ROOT / 'model/utils/pdb_reres.py' + with open(output_pdb_path, "w") as output_file: + subprocess.run([sys.executable, str(pdb_reres_path), pdb_path], + stdout=output_file, + check=True, + text=True) + + +def main(chain_ids, pdb_dir, feature_dir, stride_path, reres_path, savedir, job_index=0): + os.makedirs(savedir, exist_ok=True) + os.makedirs(os.path.join(savedir, '2d_features'), exist_ok=True) + for chain_id in chain_ids: + try: + pdb_path = os.path.join(pdb_dir, chain_id + '.pdb') + if os.path.exists(pdb_path): + features = np.load(os.path.join(feature_dir, chain_id + '.npz'))['arr_0'] + nres = features.shape[-1] + LOG.info("Processing", pdb_path) + chain = chain_id[4] + output_pdb_path = os.path.join(savedir, f"{job_index}.pdb") # this gets overwritten to save memory + file_nres = renum_pdb_file(pdb_path, reres_path, output_pdb_path) + if nres != file_nres: + with open(os.path.join(savedir, 'error.txt'), 'a') as f: + msg = f' residue number mismatch (from features) {nres}, (from pdb file) {file_nres}' + f.write(chain_id + msg + '\n') + ss_filepath = os.path.join(savedir, f'pdb_ss{job_index}.txt') # this gets overwritten to save memory + calculate_ss(output_pdb_path, chain, stride_path, ssfile=ss_filepath) + helix, strand = make_ss_matrix(ss_filepath, nres=nres) + np.savez_compressed( + os.path.join(*[savedir, '2d_features', chain_id + '.npz']), + np.stack((features, helix, strand), axis=0)) + except Exception as e: + with open(os.path.join(savedir, 'error.txt'), 'a') as f: + f.write(chain_id + str(e) + '\n') diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/scripts/chainsaw_submit.ucl_cs.sh b/scripts/chainsaw_submit.ucl_cs.sh new file mode 100644 index 0000000000000000000000000000000000000000..6f996209baf056e3765cb1b8dd498cb322f3387f --- /dev/null +++ b/scripts/chainsaw_submit.ucl_cs.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +# +# Submit chainsaw job array to SGE cluster +# +# usage: +# qsub -t 1-16933 chainsaw_submit.ucl_cs.sh +# +# Each job will expect to find an equivalent file containing a list of zipfiles, +# created by: +# +# split --suffix-length=8 --numeric-suffixes=1 --lines=1000 ../all_proteome_zipfiles.txt zipfiles. +# + +log () { + echo "[$(date)] ${1:-}" +} + +if [[ "$SGE_TASK_ID" == "undefined" ]]; then + log "Need to specify the job array details in the qsub command" + log "example: qsub -t 1-16933:1