PhoneticEngine.java
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.commons.codec.language.bm;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.EnumMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.function.Supplier;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.codec.language.bm.Languages.LanguageSet;
import org.apache.commons.codec.language.bm.Rule.Phoneme;
/**
* Converts words into potential phonetic representations.
* <p>
* This is a two-stage process. Firstly, the word is converted into a phonetic representation that takes into account the likely source language. Next, this
* phonetic representation is converted into a pan-European 'average' representation, allowing comparison between different versions of essentially the same
* word from different languages.
* </p>
* <p>
* This class is intentionally immutable and thread-safe. If you wish to alter the settings for a PhoneticEngine, you must make a new one with the updated
* settings.
* </p>
* <p>
* Ported from phoneticengine.php
* </p>
*
* @since 1.6
*/
public class PhoneticEngine {
/**
* Builder for a PhoneticEngine.
*
* @since 1.23.0
*/
public static class Builder implements Supplier<PhoneticEngine> {
/** See https://en.wikipedia.org/wiki/Hubert_Blaine_Wolfeschlegelsteinhausenbergerdorff_Sr. */
private static final int MAX_INPUT_LENGTH = 666;
private static final int MAX_PHONEMES = 20;
private boolean concat = true;
private int maxInputLength = MAX_INPUT_LENGTH;
private int maxPhonemes = MAX_PHONEMES;
private NameType nameType = NameType.GENERIC;
private RuleType ruleType = RuleType.APPROX;
private Builder() {
// empty
}
@Override
public PhoneticEngine get() {
return new PhoneticEngine(this);
}
/**
* Sets all the properties of this builder to match those of the given engine.
*
* @param engine The engine to copy properties from.
* @return This builder.
*/
public Builder setAll(final PhoneticEngine engine) {
this.nameType = engine.getNameType();
this.ruleType = engine.getRuleType();
this.concat = engine.isConcat();
this.maxPhonemes = engine.getMaxPhonemes();
return this;
}
/**
* Sets whether the engine will concatenate multiple encodings.
*
* @param concat Whether the engine will concatenate multiple encodings.
* @return This builder.
*/
public Builder setConcat(final boolean concat) {
this.concat = concat;
return this;
}
/**
* Sets the maximum input length allowed.
* <p>
* A value less than 0 will reset the maximum input length to the default value of {@value #MAX_INPUT_LENGTH}, see
* <a href="https://en.wikipedia.org/wiki/Hubert_Blaine_Wolfeschlegelsteinhausenbergerdorff_Sr.">Hubert Blaine Wolfeschlegelsteinhausenbergerdorff
* Sr.</a>.
* </p>
*
* @param maxInputLength the maximum input length allowed.
* @return This builder.
*/
public Builder setMaxInputLength(final int maxInputLength) {
this.maxInputLength = maxInputLength < 0 ? MAX_INPUT_LENGTH : maxInputLength;
return this;
}
/**
* Sets maximum number of phonemes the engine will handle.
* <p>
* A value less than 0 will reset the maximum number of phonemes to the default {@value #MAX_PHONEMES}.
* </p>
*
* @param maxPhonemes The maximum number of phonemes the engine will handle.
* @return This builder.
*/
public Builder setMaxPhonemes(final int maxPhonemes) {
this.maxPhonemes = maxPhonemes < 0 ? MAX_PHONEMES : maxPhonemes;
return this;
}
/**
* Sets the name type for the engine to be built.
* <p>
* A null value will reset the name type to the default of {@link NameType#GENERIC}.
* </p>
*
* @param nameType The type of names the engine will use.
* @return This builder.
*/
public Builder setNameType(final NameType nameType) {
this.nameType = nameType != null ? nameType : NameType.GENERIC;
return this;
}
/**
* Sets the rule type for the engine to be built.
* <p>
* A null value will reset the rule type to the default of {@link RuleType#APPROX}.
* </p>
*
* @param ruleType The type of rules the engine will use.
* @return This builder.
*/
public Builder setRuleType(final RuleType ruleType) {
this.ruleType = ruleType != null ? ruleType : RuleType.APPROX;
return this;
}
}
/**
* Manipulates a set of phonemes as they are being built up. Not intended for use outside this package, and probably not outside the {@link PhoneticEngine}
* class.
*
* @since 1.6
*/
static final class PhonemeBuilder {
/**
* An empty builder where all phonemes must come from some set of languages. This will contain a single phoneme of zero characters. This can then be
* appended to. This should be the only way to create a new phoneme from scratch.
*
* @param languages The set of languages.
* @return a new, empty phoneme builder.
*/
public static PhonemeBuilder empty(final Languages.LanguageSet languages) {
return new PhonemeBuilder(new Rule.Phoneme("", languages));
}
private final Set<Rule.Phoneme> phonemes;
private PhonemeBuilder(final Rule.Phoneme phoneme) {
this.phonemes = new LinkedHashSet<>();
this.phonemes.add(phoneme);
}
private PhonemeBuilder(final Set<Rule.Phoneme> phonemes) {
this.phonemes = phonemes;
}
/**
* Creates a new phoneme builder containing all phonemes in this one extended by {@code str}.
*
* @param str The characters to append to the phonemes.
*/
public void append(final CharSequence str) {
phonemes.forEach(ph -> ph.append(str));
}
/**
* Applies the given phoneme expression to all phonemes in this phoneme builder.
* <p>
* This will lengthen phonemes that have compatible language sets to the expression, and drop those that are incompatible.
* </p>
*
* @param phonemeExpr The expression to apply.
* @param maxPhonemes The maximum number of phonemes to build up.
*/
public void apply(final Rule.PhonemeExpr phonemeExpr, final int maxPhonemes) {
final Set<Rule.Phoneme> newPhonemes = new LinkedHashSet<>(Math.min(phonemes.size() * phonemeExpr.size(), maxPhonemes));
EXPR: for (final Rule.Phoneme left : phonemes) {
for (final Rule.Phoneme right : phonemeExpr.getPhonemes()) {
final LanguageSet languages = left.getLanguages().restrictTo(right.getLanguages());
if (!languages.isEmpty()) {
final Rule.Phoneme join = new Phoneme(left, right, languages);
if (newPhonemes.size() < maxPhonemes) {
newPhonemes.add(join);
if (newPhonemes.size() >= maxPhonemes) {
break EXPR;
}
}
}
}
}
phonemes.clear();
phonemes.addAll(newPhonemes);
}
/**
* Gets underlying phoneme set. Please don't mutate.
*
* @return the phoneme set.
*/
public Set<Rule.Phoneme> getPhonemes() {
return phonemes;
}
/**
* Stringifies the phoneme set. This produces a single string of the strings of each phoneme, joined with a pipe. This is explicitly provided in place
* of toString as it is a potentially expensive operation, which should be avoided when debugging.
*
* @return the stringified phoneme set.
*/
public String makeString() {
return phonemes.stream().map(Rule.Phoneme::getPhonemeText).collect(Collectors.joining("|"));
}
}
/**
* A function closure capturing the application of a list of rules to an input sequence at a particular offset. After invocation, the values {@code i} and
* {@code found} are updated. {@code i} points to the index of the next char in {@code input} that must be processed next (the input up to that index having
* been processed already), and {@code found} indicates if a matching rule was found or not. In the case where a matching rule was found,
* {@code phonemeBuilder} is replaced with a new builder containing the phonemes updated by the matching rule.
* <p>
* Although this class is not thread-safe (it has mutable unprotected fields), it is not shared between threads as it is constructed as needed by the
* calling methods.
* </p>
*/
private static final class RulesApplication {
private final Map<String, List<Rule>> finalRules;
private boolean found;
private int i;
private final CharSequence input;
private final int maxPhonemes;
private final PhonemeBuilder phonemeBuilder;
RulesApplication(final Map<String, List<Rule>> finalRules, final CharSequence input, final PhonemeBuilder phonemeBuilder, final int i,
final int maxPhonemes) {
this.finalRules = Objects.requireNonNull(finalRules, "finalRules");
this.phonemeBuilder = phonemeBuilder;
this.input = input;
this.i = i;
this.maxPhonemes = maxPhonemes;
}
public int getI() {
return i;
}
public PhonemeBuilder getPhonemeBuilder() {
return phonemeBuilder;
}
/**
* Invokes the rules. Loops over the rules list, stopping at the first one that has a matching context and pattern. Then applies this rule to the
* phoneme builder to produce updated phonemes. If there was no match, {@code i} is advanced one and the character is silently dropped from the phonetic
* spelling.
*
* @return {@code this}.
*/
public RulesApplication invoke() {
found = false;
int patternLength = 1;
final List<Rule> rules = finalRules.get(input.subSequence(i, i + patternLength));
if (rules != null) {
for (final Rule rule : rules) {
final String pattern = rule.getPattern();
patternLength = pattern.length();
if (rule.patternAndContextMatches(input, i)) {
phonemeBuilder.apply(rule.getPhoneme(), maxPhonemes);
found = true;
break;
}
}
}
if (!found) {
patternLength = 1;
}
i += patternLength;
return this;
}
public boolean isFound() {
return found;
}
}
private static final Map<NameType, Set<String>> NAME_PREFIXES = new EnumMap<>(NameType.class);
private static final Pattern QUOTE = Pattern.compile("'");
static {
NAME_PREFIXES.put(NameType.ASHKENAZI, Collections.unmodifiableSet(new HashSet<>(Arrays.asList("bar", "ben", "da", "de", "van", "von"))));
NAME_PREFIXES.put(NameType.SEPHARDIC, Collections.unmodifiableSet(
new HashSet<>(Arrays.asList("al", "el", "da", "dal", "de", "del", "dela", "de la", "della", "des", "di", "do", "dos", "du", "van", "von"))));
NAME_PREFIXES.put(NameType.GENERIC, Collections.unmodifiableSet(
new HashSet<>(Arrays.asList("da", "dal", "de", "del", "dela", "de la", "della", "des", "di", "do", "dos", "du", "van", "von"))));
}
/**
* Creates a new builder for a PhoneticEngine.
*
* @return a new builder for a PhoneticEngine.
* @since 1.23.0
*/
public static Builder builder() {
return new Builder();
}
/**
* Joins some strings with an internal separator.
*
* @param strings Strings to join.
* @param sep String to separate them with.
* @return A single String consisting of each element of {@code strings} interleaved by {@code sep}.
*/
private static String join(final List<String> strings, final String sep) {
return strings.stream().collect(Collectors.joining(sep));
}
private final boolean concat;
private final Lang lang;
private final int maxInputLength;
private final int maxPhonemes;
private final NameType nameType;
private final RuleType ruleType;
/**
* Creates a new, fully-configured phonetic engine.
*
* @param builder The builder to use for configuration.
* @throws IllegalArgumentException if ruleType is RULES.
*/
private PhoneticEngine(final Builder builder) {
if (builder.ruleType == RuleType.RULES) {
throw new IllegalArgumentException("ruleType must not be " + RuleType.RULES);
}
this.nameType = builder.nameType;
this.ruleType = builder.ruleType;
this.concat = builder.concat;
this.lang = Lang.instance(builder.nameType);
this.maxPhonemes = builder.maxPhonemes;
this.maxInputLength = builder.maxInputLength;
}
/**
* Generates a new, fully-configured phonetic engine.
*
* @param nameType the type of names it will use, null is treated as {@link NameType#GENERIC}.
* @param ruleType the type of rules it will apply, null is treated as {@link RuleType#APPROX}.
* @param concatenate if it will concatenate multiple encodings.
* @deprecated Use {@link #builder()} instead.
*/
@Deprecated
public PhoneticEngine(final NameType nameType, final RuleType ruleType, final boolean concatenate) {
this(nameType, ruleType, concatenate, Builder.MAX_PHONEMES);
}
/**
* Generates a new, fully-configured phonetic engine.
*
* @param nameType the type of names it will use, null is treated as {@link NameType#GENERIC}.
* @param ruleType the type of rules it will apply, null is treated as {@link RuleType#APPROX}.
* @param concatenate if it will concatenate multiple encodings.
* @param maxPhonemes the maximum number of phonemes that will be handled, less than 0 will reset to the default of {@value Builder#MAX_PHONEMES}.
* @throws IllegalArgumentException if ruleType is RULES.
* @since 1.7
* @deprecated Use {@link #builder()} instead.
*/
@Deprecated
public PhoneticEngine(final NameType nameType, final RuleType ruleType, final boolean concatenate, final int maxPhonemes) {
this(builder().setNameType(nameType).setRuleType(ruleType).setConcat(concatenate).setMaxPhonemes(maxPhonemes)
.setMaxInputLength(Builder.MAX_INPUT_LENGTH));
}
/**
* Applies the final rules to convert from a language-specific phonetic representation to a language-independent representation.
*
* @param phonemeBuilder The current phonemes.
* @param finalRules The final rules to apply.
* @return The resulting phonemes.
*/
private PhonemeBuilder applyFinalRules(final PhonemeBuilder phonemeBuilder, final Map<String, List<Rule>> finalRules) {
Objects.requireNonNull(finalRules, "finalRules");
if (finalRules.isEmpty()) {
return phonemeBuilder;
}
final Map<Rule.Phoneme, Rule.Phoneme> phonemes = new TreeMap<>(Rule.Phoneme.COMPARATOR);
phonemeBuilder.getPhonemes().forEach(phoneme -> {
PhonemeBuilder subBuilder = PhonemeBuilder.empty(phoneme.getLanguages());
final CharSequence phonemeText = phoneme.getPhonemeText();
final int length = phonemeText.length();
for (int i = 0; i < length;) {
final RulesApplication rulesApplication = new RulesApplication(finalRules, phonemeText, subBuilder, i, maxPhonemes).invoke();
final boolean found = rulesApplication.isFound();
subBuilder = rulesApplication.getPhonemeBuilder();
if (!found) {
// not found, appending as-is
subBuilder.append(phonemeText.subSequence(i, i + 1));
}
i = rulesApplication.getI();
}
// the phonemes map orders the phonemes only based on their text, but ignores the language set
// when adding new phonemes, check for equal phonemes and merge their language set, otherwise
// phonemes with the same text but different language set get lost
subBuilder.getPhonemes().forEach(newPhoneme -> {
if (phonemes.containsKey(newPhoneme)) {
final Rule.Phoneme oldPhoneme = phonemes.remove(newPhoneme);
final Rule.Phoneme mergedPhoneme = oldPhoneme.mergeWithLanguage(newPhoneme.getLanguages());
phonemes.put(mergedPhoneme, mergedPhoneme);
} else {
phonemes.put(newPhoneme, newPhoneme);
}
});
});
return new PhonemeBuilder(phonemes.keySet());
}
/**
* Encodes a string to its phonetic representation.
*
* @param input the String to encode, not null.
* @return The encoding of the input.
* @throws IllegalArgumentException if the input is longer than the maximum allowed length.
*/
public String encode(final String input) {
return encode(input, lang.guessLanguages(input));
}
/**
* Encodes an input string into an output phonetic representation, given a set of possible origin languages.
*
* @param input String to phoneticise; a String with dashes or spaces separating each word, not null.
* @param languageSet set of possible origin languages.
* @return A phonetic representation of the input; a String containing '-'-separated phonetic representations of the input.
* @throws IllegalArgumentException if the input is longer than the maximum allowed length.
*/
public String encode(String input, final Languages.LanguageSet languageSet) {
if (input.length() > maxInputLength) {
throw new IllegalArgumentException("Input is greater than maxInputLength (" + maxInputLength + ").");
}
final Map<String, List<Rule>> rules = Rule.getInstanceMap(this.nameType, RuleType.RULES, languageSet);
// rules common across many (all) languages
final Map<String, List<Rule>> finalRules1 = Rule.getInstanceMap(this.nameType, this.ruleType, "common");
// rules that apply to a specific language that may be ambiguous or wrong if applied to other languages
final Map<String, List<Rule>> finalRules2 = Rule.getInstanceMap(this.nameType, this.ruleType, languageSet);
// tidy the input
// lower case is a locale-dependent operation
input = input.toLowerCase(Locale.ENGLISH).replace('-', ' ').trim();
if (this.nameType == NameType.GENERIC) {
final String dQuotePrefix = "d'";
final int dqpLen = dQuotePrefix.length();
if (input.startsWith(dQuotePrefix)) { // check for d'
String remainder = input.substring(dqpLen);
// Find remainder without allocating new string.
final int start = lastRepeat(remainder, dQuotePrefix, dqpLen);
remainder = remainder.substring(start);
final String combined = "d" + remainder;
return "(" + encode(remainder) + ")-(" + encode(combined) + ")";
}
for (final String l : NAME_PREFIXES.get(this.nameType)) {
// handle generic prefixes
if (input.startsWith(l + " ")) {
// check for any prefix in the words list
final String remainder = input.substring(l.length() + 1); // input without the prefix
final String combined = l + remainder; // input with prefix without space
return "(" + encode(remainder) + ")-(" + encode(combined) + ")";
}
}
}
final List<String> words = Arrays.asList(ResourceConstants.SPACES.split(input));
final List<String> words2 = new ArrayList<>();
// special-case handling of word prefixes based upon the name type
switch (this.nameType) {
case SEPHARDIC:
words.forEach(aWord -> {
final String[] parts = QUOTE.split(aWord, -1);
words2.add(parts[parts.length - 1]);
});
words2.removeAll(NAME_PREFIXES.get(this.nameType));
break;
case ASHKENAZI:
words2.addAll(words);
words2.removeAll(NAME_PREFIXES.get(this.nameType));
break;
case GENERIC:
words2.addAll(words);
break;
default:
throw new IllegalStateException("Unreachable case: " + this.nameType);
}
if (this.concat) {
// concat mode enabled
input = join(words2, " ");
} else if (words2.size() == 1) {
// not a multi-word name
input = words.iterator().next();
} else if (!words2.isEmpty()) {
// encode each word in a multi-word name separately (normally used for approx matches)
final StringBuilder result = new StringBuilder();
words2.forEach(word -> result.append("-").append(encode(word)));
// return the result without the leading "-"
return result.substring(1);
}
PhonemeBuilder phonemeBuilder = PhonemeBuilder.empty(languageSet);
// loop over each char in the input - we will handle the increment manually
for (int i = 0; i < input.length();) {
final RulesApplication rulesApplication = new RulesApplication(rules, input, phonemeBuilder, i, maxPhonemes).invoke();
i = rulesApplication.getI();
phonemeBuilder = rulesApplication.getPhonemeBuilder();
}
// Apply the general rules
phonemeBuilder = applyFinalRules(phonemeBuilder, finalRules1);
// Apply the language-specific rules
phonemeBuilder = applyFinalRules(phonemeBuilder, finalRules2);
return phonemeBuilder.makeString();
}
/**
* Gets the Lang language guessing rules being used.
*
* @return The Lang in use.
*/
public Lang getLang() {
return this.lang;
}
/**
* Gets the maximum number of phonemes the engine will calculate for a given input.
*
* @return The maximum number of phonemes.
* @since 1.7
*/
public int getMaxPhonemes() {
return this.maxPhonemes;
}
/**
* Gets the NameType being used.
*
* @return The NameType in use.
*/
public NameType getNameType() {
return this.nameType;
}
/**
* Gets the RuleType being used.
*
* @return The RuleType in use.
*/
public RuleType getRuleType() {
return this.ruleType;
}
/**
* Gets if multiple phonetic encodings are concatenated or if just the first one is kept.
*
* @return true if multiple phonetic encodings are returned, false if just the first is.
*/
public boolean isConcat() {
return this.concat;
}
/**
* Finds the index past the last occurrence of a repeating prefix in a string, starting from the beginning of the string and moving forward.
*
* @param source The source string to search within.
* @param prefix The prefix to look for.
* @param prefixLen The length of the prefix.
* @return The index in the source string where the last occurrence of the repeating prefix ends.
*/
private int lastRepeat(final String source, final String prefix, final int prefixLen) {
// Find without allocating new string.
int start = 0;
while (start + prefixLen <= source.length()) {
int i = 0;
while (i < prefixLen && source.charAt(start + i) == prefix.charAt(i)) {
i++;
}
if (i != prefixLen) {
break;
}
start += prefixLen;
}
return start;
}
}