| |
| |
| |
| |
| |
| |
|
|
| from argparse import ArgumentParser |
| import math |
| import os |
| from random import randint |
| import sys, gzip |
|
|
|
|
| def count_ngrams(snt, max_n): |
| """ |
| Return a dictionary of ngram counts (up to length /max_n/) |
| for sentence (list of words) /snt/. |
| """ |
| ret = {} |
| for i in xrange(len(snt)): |
| for k in xrange(i + 1, min(i + max_n + 1, len(snt) + 1)): |
| key = tuple(snt[i:k]) |
| ret[key] = ret.get(key, 0) + 1 |
| return ret |
|
|
|
|
| def max_counts(ng1, ng2): |
| """ |
| Return a dicitonary of ngram counts such that |
| each count is the greater of the two individual counts |
| for each ngram in the input ngram count dictionaries |
| /ng1/ and /ng2/. |
| """ |
| ret = ng1.copy() |
| for k, v in ng2.items(): |
| ret[k] = max(ret.get(k, 0), v) |
| return ret |
|
|
|
|
| def ng_hits(hyp, ref, max_n): |
| """ |
| Return a list of ngram counts such that each ngram count |
| is the minimum of the counts in hyp and ref, up to ngram |
| length /max_n/. |
| """ |
| ret = [0 for i in xrange(max_n)] |
| for ng, cnt in hyp.items(): |
| k = ng |
| if len(k) <= max_n: |
| ret[len(k) - 1] += min(cnt, ref.get(ng, 0)) |
| return ret |
|
|
|
|
| class BleuScore: |
| def __init__(self, hyp, ref, max_n=4, bootstrap=1000): |
| |
| self.hits = [ |
| ng_hits(hyp.ngrams[i], ref.ngrams[i], max_n) |
| for i in xrange(len(hyp.ngrams))] |
| self.max_n = max_n |
| self.hyp = hyp |
| self.ref = ref |
| self.lower = None |
| self.upper = None |
| self.median = None |
| self.actual = self.score([i for i in xrange(len(hyp.snt))]) |
| if bootstrap: |
| self.bootstrap = [self.score([randint(0, len(hyp.snt) - 1) |
| for s in hyp.snt]) |
| for i in xrange(bootstrap)] |
| self.bootstrap.sort() |
| else: |
| self.bootstrap = [self.actual] |
| pass |
|
|
| def score(self, sample): |
| hits = [0 for i in xrange(self.max_n)] |
| self.hyplen = 0 |
| self.reflen = 0 |
| self.total = [0 for i in hits] |
| for i in sample: |
| self.hyplen += len(self.hyp.snt[i]) |
| self.reflen += len(self.ref.snt[i]) |
| for n in xrange(self.max_n): |
| hits[n] += self.hits[i][n] |
| self.total[n] += max(len(self.hyp.snt[i]) - n, 0) |
| pass |
| self.prec = [float(hits[n]) / self.total[n] |
| for n in xrange(self.max_n)] |
| ret = sum([math.log(x) for x in self.prec]) / self.max_n |
| self.BP = min( |
| 1, math.exp(1. - float(self.reflen) / float(self.hyplen))) |
| ret += math.log(self.BP) |
| return math.exp(ret) |
|
|
|
|
| class Document: |
| def __init__(self, fname=None): |
| self.fname = fname |
| if fname: |
| if fname[-3:] == ".gz": |
| self.snt = [line.strip().split() for line in gzip.open(fname).readlines()] |
| else: |
| self.snt = [line.strip().split() for line in open(fname)] |
| pass |
| self.ngrams = [count_ngrams(snt, 4) for snt in self.snt] |
| |
| else: |
| self.snt = None |
| self.ngrams = None |
|
|
| def merge(self, R): |
| self.fname = "multi-ref" |
| self.ngrams = [x for x in R[0].ngrams] |
| self.snt = [x for x in R[0].snt] |
| for i in xrange(len(R[0].ngrams)): |
| for k in xrange(1, len(R)): |
| self.ngrams[i] = max_counts(self.ngrams[i], R[k].ngrams[i]) |
|
|
| def update(self, hyp, R): |
| for i, hyp_snt in enumerate(hyp.snt): |
| clen = len(hyp_snt) |
| K = 0 |
| for k in xrange(1, len(R)): |
| k_snt = R[k].snt[i] |
| assert len(R[k].snt) == len(hyp.snt), ( |
| "Mismatch in number of sentences " + |
| "between reference and candidate") |
| if abs(len(k_snt) - clen) == abs(len(R[K].snt[i]) - clen): |
| if len(k_snt) < len(R[K].snt[i]): |
| K = k |
| elif abs(len(k_snt) - clen) < abs(len(R[K].snt[i]) - clen): |
| K = k |
| self.snt[i] = R[K].snt[i] |
|
|
|
|
| if __name__ == "__main__": |
| argparser = ArgumentParser() |
| argparser.add_argument( |
| "-r", "--ref", nargs='+', help="Reference translation(s).") |
| argparser.add_argument( |
| "-c", "--cand", nargs='+', help="Candidate translations.") |
| argparser.add_argument( |
| "-i", "--individual", action='store_true', |
| help="Compute BLEU scores for individual references.") |
| argparser.add_argument( |
| "-b", "--bootstrap", type=int, default=1000, |
| help="Sample size for bootstrap resampling.") |
| argparser.add_argument( |
| "-a", "--alpha", type=float, default=.05, |
| help="1-alpha = confidence interval.") |
| args = argparser.parse_args(sys.argv[1:]) |
| R = [Document(fname) for fname in args.ref] |
| C = [Document(fname) for fname in args.cand] |
| Rx = Document() |
| Rx.merge(R) |
| for c in C: |
| |
| Rx.update(c, R) |
| bleu = BleuScore(c, Rx, bootstrap=args.bootstrap) |
| print "%5.2f %s [%5.2f-%5.2f; %5.2f] %s" % ( |
| 100 * bleu.actual, |
| os.path.basename(Rx.fname), |
| 100 * bleu.bootstrap[int((args.alpha / 2) * args.bootstrap)], |
| 100 * bleu.bootstrap[int((1 - (args.alpha / 2)) * args.bootstrap)], |
| 100 * bleu.bootstrap[int(.5 * args.bootstrap)], |
| c.fname) |
|
|
| if args.individual: |
| for r in R: |
| bleu = BleuScore(c, r, bootstrap=args.bootstrap) |
| print " %5.2f %s" % ( |
| 100 * bleu.actual, os.path.basename(r.fname)) |
| |
|
|
| |
| |
| |
|
|
