1// Copyright (C) 2016 The Qt Company Ltd.
2// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
3
4#include "simtexth.h"
5#include "translator.h"
6
7#include <QtCore/QByteArray>
8#include <QtCore/QString>
9#include <QtCore/QList>
10
11
12QT_BEGIN_NAMESPACE
13
14typedef QList<TranslatorMessage> TML;
15
16/*
17 How similar are two texts? The approach used here relies on co-occurrence
18 matrices and is very efficient.
19
20 Let's see with an example: how similar are "here" and "hither"? The
21 co-occurrence matrix M for "here" is M[h,e] = 1, M[e,r] = 1, M[r,e] = 1, and 0
22 elsewhere; the matrix N for "hither" is N[h,i] = 1, N[i,t] = 1, ...,
23 N[h,e] = 1, N[e,r] = 1, and 0 elsewhere. The union U of both matrices is the
24 matrix U[i,j] = max { M[i,j], N[i,j] }, and the intersection V is
25 V[i,j] = min { M[i,j], N[i,j] }. The score for a pair of texts is
26
27 score = (sum of V[i,j] over all i, j) / (sum of U[i,j] over all i, j),
28
29 a formula suggested by Arnt Gulbrandsen. Here we have
30
31 score = 2 / 6,
32
33 or one third.
34
35 The implementation differs from this in a few details. Most importantly,
36 repetitions are ignored; for input "xxx", M[x,x] equals 1, not 2.
37*/
38
39/*
40 Every character is assigned to one of 20 buckets so that the co-occurrence
41 matrix requires only 20 * 20 = 400 bits, not 256 * 256 = 65536 bits or even
42 more if we want the whole Unicode. Which character falls in which bucket is
43 arbitrary.
44
45 The second half of the table is a replica of the first half, because of
46 laziness.
47*/
48static const int indexOf[256] = {
49 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
50 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
51// ! " # $ % & ' ( ) * + , - . /
52 0, 2, 6, 7, 10, 12, 15, 19, 2, 6, 7, 10, 12, 15, 19, 0,
53// 0 1 2 3 4 5 6 7 8 9 : ; < = > ?
54 1, 3, 4, 5, 8, 9, 11, 13, 14, 16, 2, 6, 7, 10, 12, 15,
55// @ A B C D E F G H I J K L M N O
56 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14,
57// P Q R S T U V W X Y Z [ \ ] ^ _
58 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0,
59// ` a b c d e f g h i j k l m n o
60 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14,
61// p q r s t u v w x y z { | } ~
62 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0,
63
64 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
65 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
66 0, 2, 6, 7, 10, 12, 15, 19, 2, 6, 7, 10, 12, 15, 19, 0,
67 1, 3, 4, 5, 8, 9, 11, 13, 14, 16, 2, 6, 7, 10, 12, 15,
68 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14,
69 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0,
70 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 6, 10, 11, 12, 13, 14,
71 15, 12, 16, 17, 18, 19, 2, 10, 15, 7, 19, 2, 6, 7, 10, 0
72};
73
74/*
75 The entry bitCount[i] (for i between 0 and 255) is the number of bits used to
76 represent i in binary.
77*/
78static const int bitCount[256] = {
79 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
80 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
81 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
82 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
83 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
84 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
85 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
86 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
87 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
88 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
89 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
90 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
91 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
92 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
93 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
94 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
95};
96
97static inline void setCoOccurence(CoMatrix &m, char c, char d)
98{
99 int k = indexOf[(uchar) c] + 20 * indexOf[(uchar) d];
100 m.b[k >> 3] |= (1 << (k & 0x7));
101}
102
103CoMatrix::CoMatrix(const QString &str)
104{
105 QByteArray ba = str.toUtf8();
106 const char *text = ba.constData();
107 char c = '\0', d;
108 memset( s: b, c: 0, n: 52 );
109 /*
110 The Knuth books are not in the office only for show; they help make
111 loops 30% faster and 20% as readable.
112 */
113 while ( (d = *text) != '\0' ) {
114 setCoOccurence(m&: *this, c, d);
115 if ( (c = *++text) != '\0' ) {
116 setCoOccurence(m&: *this, c: d, d: c);
117 text++;
118 }
119 }
120}
121
122static inline int worth(const CoMatrix &m)
123{
124 int w = 0;
125 for (int i = 0; i < 50; i++)
126 w += bitCount[m.b[i]];
127 return w;
128}
129
130static inline CoMatrix reunion(const CoMatrix &m, const CoMatrix &n)
131{
132 CoMatrix p;
133 for (int i = 0; i < 13; ++i)
134 p.w[i] = m.w[i] | n.w[i];
135 return p;
136}
137
138static inline CoMatrix intersection(const CoMatrix &m, const CoMatrix &n)
139{
140 CoMatrix p;
141 for (int i = 0; i < 13; ++i)
142 p.w[i] = m.w[i] & n.w[i];
143 return p;
144}
145
146StringSimilarityMatcher::StringSimilarityMatcher(const QString &stringToMatch)
147 : m_cm(stringToMatch)
148{
149 m_length = stringToMatch.size();
150}
151
152int StringSimilarityMatcher::getSimilarityScore(const QString &strCandidate)
153{
154 CoMatrix cmTarget(strCandidate);
155 int delta = qAbs(t: m_length - strCandidate.size());
156 int score = ( (worth(m: intersection(m: m_cm, n: cmTarget)) + 1) << 10 ) /
157 ( worth(m: reunion(m: m_cm, n: cmTarget)) + (delta << 1) + 1 );
158 return score;
159}
160
161CandidateList similarTextHeuristicCandidates(const Translator *tor,
162 const QString &text, int maxCandidates)
163{
164 QList<int> scores;
165 CandidateList candidates;
166 StringSimilarityMatcher matcher(text);
167
168 for (const TranslatorMessage &mtm : tor->messages()) {
169 if (mtm.type() == TranslatorMessage::Unfinished
170 || mtm.translation().isEmpty())
171 continue;
172
173 QString s = mtm.sourceText();
174 int score = matcher.getSimilarityScore(strCandidate: s);
175
176 if (candidates.size() == maxCandidates && score > scores[maxCandidates - 1] )
177 candidates.removeLast();
178
179 if (candidates.size() < maxCandidates && score >= textSimilarityThreshold) {
180 Candidate cand(mtm.context(), s, mtm.comment(), mtm.translation());
181
182 int i;
183 for (i = 0; i < candidates.size(); i++) {
184 if (score >= scores.at(i)) {
185 if (score == scores.at(i)) {
186 if (candidates.at(i) == cand)
187 goto continue_outer_loop;
188 } else {
189 break;
190 }
191 }
192 }
193 scores.insert(i, t: score);
194 candidates.insert(i, t: cand);
195 }
196 continue_outer_loop:
197 ;
198 }
199 return candidates;
200}
201
202QT_END_NAMESPACE
203

source code of qttools/src/linguist/shared/simtexth.cpp