| import re |
|
|
| def postcor(blist): |
| crlist = blist |
| remrow = [] |
| for i in range(len(crlist)-1): |
| if len(crlist[i][0]) <= 1 or len(crlist[i][1]) <= 1 or crlist[i][2] == "other": |
| remrow += [crlist[i]] |
| continue |
| xlt = re.findall(r'[a-zA-Z]', crlist[i][0]) |
| if len(xlt)==0: |
| remrow += [crlist[i]] |
| continue |
| xlt = re.findall(r'[a-zA-Z]', crlist[i][1]) |
| if len(xlt) == 0: |
| remrow += [crlist[i]] |
| continue |
| |
|
|
| for j in range(i+1,len(crlist)): |
| if re.sub(r"(-|'| |_)", "", crlist[i][0]).lower() == re.sub(r"(-|'| |_)", "", crlist[j][0]).lower(): |
| if len(crlist[i][0]) < len(crlist[j][0]): |
| crlist[j][0] = crlist[i][0] |
| elif len(crlist[i][0]) > len(crlist[j][0]): |
| crlist[i][0] = crlist[j][0] |
|
|
| if re.sub(r"(-|'| |_)", "", crlist[i][1]).lower() == re.sub(r"(-|'| |_)", "", crlist[j][1]).lower(): |
| if len(crlist[i][1]) < len(crlist[j][1]): |
| crlist[j][1] = crlist[i][1] |
| elif len(crlist[i][1]) > len(crlist[j][1]): |
| crlist[i][1] = crlist[j][1] |
|
|
| if len(crlist[i][0])-len(crlist[j][0]) == 1 and crlist[j][0] in crlist[i][0] and crlist[i][0][-1] == "s": |
| crlist[i][0] = crlist[j][0] |
| elif len(crlist[i][0])-len(crlist[j][0]) == -1 and crlist[i][0] in crlist[j][0] and crlist[j][0][-1] == "s": |
| crlist[j][0] = crlist[i][0] |
| if len(crlist[i][1])-len(crlist[j][1]) == 1 and crlist[j][1] in crlist[i][1] and crlist[i][1][-1] == "s": |
| crlist[i][0] = crlist[j][0] |
| elif len(crlist[i][1])-len(crlist[j][1]) == -1 and crlist[i][1] in crlist[j][1] and crlist[j][1][-1] == "s": |
| crlist[j][1] = crlist[i][1] |
|
|
| for rw in remrow: |
| crlist.remove(rw) |
| |
| return crlist |
| |
| |
| def precor(text): |
| lines = text |
| lines = lines.replace("breast and prostate cancer","breast cancer and prostate cancer") |
| lines = lines.replace("prostate and breast cancer","prostate cancer and breast cancer") |
| lines = lines.replace("breast, prostate and ovarian cancer","breast cancer, prostate cancer and ovarian cancer") |
| lines = re.sub(r"\[\d*\]", "", lines) |
| lines = re.sub(r'\(\s?(figure|Figure|table|Table|fig\.|Fig\.|tab\.|Tab\.)(\s?\w)*\s?\)',"", lines) |
| lines = re.sub(r'www\.(?:[-\w.]|(?:%[\da-fA-F]{2}))+', "", lines) |
| lines = re.sub(r'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+', "", lines) |
| lines = re.sub(r'[a-zA-Z0-9]+[_\-.]*[a-zA-Z0-9]*@[a-zA-Z0-9]+\.\w+', "", lines) |
| lines = re.sub(r'\(\)', "", lines) |
| lines = re.sub(r'\[\s?[0-9]+(\–|,|\-)\s?[0-9]+\s?\]', "", lines) |
| lines = re.sub(r'\(\s?[0-9]+(\–|,|\-)\s?[0-9]+\s?\)', "", lines) |
| lines = re.sub(r'\[\s?[0-9]+(\–|,|\-)?\s?[0-9]*\s?\]', "", lines) |
| lines = re.sub(r'\(\s?[0-9]+(\–|,|\-)?\s?[0-9]*\s?\)', "", lines) |
| punc = ";.,?([)]" |
| for e in punc: |
| lines = lines.replace(e, " "+e+" ") |
| return lines |
