tesseract  4.1.1
tesseract::Validator Class Referenceabstract

#include <validator.h>

Inheritance diagram for tesseract::Validator:
tesseract::ValidateGrapheme tesseract::ValidateIndic tesseract::ValidateJavanese tesseract::ValidateKhmer tesseract::ValidateMyanmar

Public Member Functions

virtual ~Validator ()
 

Static Public Member Functions

static bool ValidateCleanAndSegment (GraphemeNormMode g_mode, bool report_errors, const std::vector< char32 > &src, std::vector< std::vector< char32 >> *dest)
 
static bool IsZeroWidthMark (char32 ch)
 

Static Public Attributes

static const char32 kZeroWidthSpace = 0x200B
 
static const char32 kZeroWidthNonJoiner = 0x200C
 
static const char32 kZeroWidthJoiner = 0x200D
 
static const char32 kLeftToRightMark = 0x200E
 
static const char32 kRightToLeftMark = 0x200F
 
static const char32 kInvalid = 0xfffd
 

Protected Types

enum  CharClass {
  CharClass::kConsonant = 'C', CharClass::kVowel = 'V', CharClass::kVirama = 'H', CharClass::kMatra = 'M',
  CharClass::kMatraPiece = 'P', CharClass::kVowelModifier = 'D', CharClass::kZeroWidthNonJoiner = 'z', CharClass::kZeroWidthJoiner = 'Z',
  CharClass::kVedicMark = 'v', CharClass::kNukta = 'N', CharClass::kRobat = 'R', CharClass::kOther = 'O',
  CharClass::kWhitespace = ' ', CharClass::kCombiner = 'c'
}
 
using IndicPair = std::pair< CharClass, char32 >
 

Protected Member Functions

 Validator (ViramaScript script, bool report_errors)
 
bool ValidateCleanAndSegmentInternal (GraphemeNormMode g_mode, const std::vector< char32 > &src, std::vector< std::vector< char32 >> *dest)
 
void MoveResultsToDest (GraphemeNormMode g_mode, std::vector< std::vector< char32 >> *dest)
 
bool IsSubscriptScript () const
 
bool CodeOnlyToOutput ()
 
void MultiCodePart (unsigned length)
 
bool UseMultiCode (unsigned length)
 
virtual bool ConsumeGraphemeIfValid ()=0
 
void ComputeClassCodes (const std::vector< char32 > &text)
 
virtual CharClass UnicodeToCharClass (char32 ch) const =0
 
void Clear ()
 

Static Protected Member Functions

static std::unique_ptr< ValidatorScriptValidator (ViramaScript script, bool report_errors)
 
static ViramaScript MostFrequentViramaScript (const std::vector< char32 > &utf32)
 
static bool IsVirama (char32 unicode)
 
static bool IsVedicAccent (char32 unicode)
 

Protected Attributes

ViramaScript script_
 
std::vector< IndicPaircodes_
 
std::vector< std::vector< char32 > > parts_
 
std::vector< char32output_
 
unsigned codes_used_
 
unsigned output_used_
 
bool report_errors_
 

Static Protected Attributes

static const int kIndicCodePageSize = 128
 
static const char32 kMinIndicUnicode = 0x900
 
static const char32 kMaxSinhalaUnicode = 0xdff
 
static const char32 kMaxViramaScriptUnicode = 0x17ff
 
static const char32 kSinhalaVirama = 0xdca
 
static const char32 kMyanmarVirama = 0x1039
 
static const char32 kKhmerVirama = 0x17d2
 
static const char32 kJavaneseVirama = 0xa9c0
 
static const char32 kMaxJavaneseUnicode = 0xa9df
 

Detailed Description

Definition at line 71 of file validator.h.

Member Typedef Documentation

◆ IndicPair

using tesseract::Validator::IndicPair = std::pair<CharClass, char32>
protected

Definition at line 133 of file validator.h.

Member Enumeration Documentation

◆ CharClass

enum tesseract::Validator::CharClass
strongprotected
Enumerator
kConsonant 
kVowel 
kVirama 
kMatra 
kMatraPiece 
kVowelModifier 
kZeroWidthNonJoiner 
kZeroWidthJoiner 
kVedicMark 
kNukta 
kRobat 
kOther 
kWhitespace 
kCombiner 

Definition at line 111 of file validator.h.

111  {
112  // NOTE: The values of the enum members are meaningless and arbitrary, ie
113  // they are not used for sorting, or any other risky application.
114  // The reason they are what they are is they are a single character
115  // abbreviation that can be used in a regexp/BNF definition of a grammar,
116  // IN A COMMENT, and still not relied upon in the code.
117  kConsonant = 'C',
118  kVowel = 'V',
119  kVirama = 'H', // (aka Halant)
120  kMatra = 'M', // (aka Dependent Vowel)
121  kMatraPiece = 'P', // unicode provides pieces of Matras.
122  kVowelModifier = 'D', // (candrabindu, anusvara, visarga, other marks)
123  kZeroWidthNonJoiner = 'z', // Unicode Zero Width Non-Joiner U+200C
124  kZeroWidthJoiner = 'Z', // Unicode Zero Width Joiner U+200D
125  kVedicMark = 'v', // Modifiers can come modify any indic syllable.
126  kNukta = 'N', // Occurs only immediately after consonants.
127  kRobat = 'R', // Khmer only.
128  kOther = 'O', // (digits, measures, non-Indic, etc)
129  // Additional classes used only by ValidateGrapheme.
130  kWhitespace = ' ',
131  kCombiner = 'c', // Combiners other than virama.
132  };
static const char32 kZeroWidthNonJoiner
Definition: validator.h:95
static const char32 kZeroWidthJoiner
Definition: validator.h:96

Constructor & Destructor Documentation

◆ ~Validator()

tesseract::Validator::~Validator ( )
virtualdefault

◆ Validator()

tesseract::Validator::Validator ( ViramaScript  script,
bool  report_errors 
)
inlineprotected

Definition at line 135 of file validator.h.

136  : script_(script),
137  codes_used_(0),
138  output_used_(0),
139  report_errors_(report_errors) {}
unsigned codes_used_
Definition: validator.h:237
unsigned output_used_
Definition: validator.h:239
ViramaScript script_
Definition: validator.h:229

Member Function Documentation

◆ Clear()

void tesseract::Validator::Clear ( )
protected

Definition at line 214 of file validator.cpp.

214  {
215  codes_.clear();
216  parts_.clear();
217  output_.clear();
218  codes_used_ = 0;
219  output_used_ = 0;
220 }
unsigned codes_used_
Definition: validator.h:237
std::vector< std::vector< char32 > > parts_
Definition: validator.h:233
unsigned output_used_
Definition: validator.h:239
std::vector< char32 > output_
Definition: validator.h:235
std::vector< IndicPair > codes_
Definition: validator.h:231

◆ CodeOnlyToOutput()

bool tesseract::Validator::CodeOnlyToOutput ( )
inlineprotected

Definition at line 171 of file validator.h.

171  {
172  output_.push_back(codes_[codes_used_].second);
173  return ++codes_used_ == codes_.size();
174  }
unsigned codes_used_
Definition: validator.h:237
std::vector< char32 > output_
Definition: validator.h:235
std::vector< IndicPair > codes_
Definition: validator.h:231

◆ ComputeClassCodes()

void tesseract::Validator::ComputeClassCodes ( const std::vector< char32 > &  text)
protected

Definition at line 206 of file validator.cpp.

206  {
207  codes_.reserve(text.size());
208  for (char32 c : text) {
209  codes_.push_back(std::make_pair(UnicodeToCharClass(c), c));
210  }
211 }
virtual CharClass UnicodeToCharClass(char32 ch) const =0
signed int char32
std::vector< IndicPair > codes_
Definition: validator.h:231

◆ ConsumeGraphemeIfValid()

virtual bool tesseract::Validator::ConsumeGraphemeIfValid ( )
protectedpure virtual

◆ IsSubscriptScript()

◆ IsVedicAccent()

bool tesseract::Validator::IsVedicAccent ( char32  unicode)
staticprotected

Definition at line 191 of file validator.cpp.

191  {
192  return (0x1cd0 <= unicode && unicode < 0x1d00) ||
193  (0xa8e0 <= unicode && unicode <= 0xa8f7) ||
194  (0x951 <= unicode && unicode <= 0x954);
195 }

◆ IsVirama()

bool tesseract::Validator::IsVirama ( char32  unicode)
staticprotected

Definition at line 180 of file validator.cpp.

180  {
181  return (kMinIndicUnicode <= unicode && unicode <= kMaxSinhalaUnicode &&
182  (unicode & 0x7f) == 0x4d) ||
183  unicode == kSinhalaVirama ||
184  unicode == kJavaneseVirama ||
185  unicode == kMyanmarVirama ||
186  unicode == kKhmerVirama;
187 }
static const char32 kMinIndicUnicode
Definition: validator.h:215
static const char32 kKhmerVirama
Definition: validator.h:223
static const char32 kSinhalaVirama
Definition: validator.h:221
static const char32 kMaxSinhalaUnicode
Definition: validator.h:217
static const char32 kJavaneseVirama
Definition: validator.h:225
static const char32 kMyanmarVirama
Definition: validator.h:222

◆ IsZeroWidthMark()

static bool tesseract::Validator::IsZeroWidthMark ( char32  ch)
inlinestatic

Definition at line 87 of file validator.h.

87  {
88  return ch == kZeroWidthSpace || ch == kLeftToRightMark ||
89  ch == kRightToLeftMark || ch == kInvalid;
90  }
static const char32 kInvalid
Definition: validator.h:99
static const char32 kRightToLeftMark
Definition: validator.h:98
static const char32 kZeroWidthSpace
Definition: validator.h:94
static const char32 kLeftToRightMark
Definition: validator.h:97

◆ MostFrequentViramaScript()

ViramaScript tesseract::Validator::MostFrequentViramaScript ( const std::vector< char32 > &  utf32)
staticprotected

Definition at line 143 of file validator.cpp.

144  {
145  std::unordered_map<int, int> histogram;
146  for (char32 ch : utf32) {
147  // Determine the codepage base. For the Indic scripts, Khmer and Javanese, it is
148  // sufficient to divide by kIndicCodePageSize but Myanmar is all over the
149  // unicode code space, so use its script id.
150  int base = ch / kIndicCodePageSize;
151  IcuErrorCode err;
152  UScriptCode script_code = uscript_getScript(ch, err);
153  if ((kMinIndicUnicode <= ch && ch <= kMaxJavaneseUnicode &&
154  script_code != USCRIPT_COMMON) ||
155  script_code == USCRIPT_MYANMAR) {
156  if (script_code == USCRIPT_MYANMAR)
157  base = static_cast<char32>(ViramaScript::kMyanmar) / kIndicCodePageSize;
158  ++histogram[base];
159  }
160  }
161  if (!histogram.empty()) {
162  int base =
163  std::max_element(histogram.begin(), histogram.end(), CmpPairSecond)
164  ->first;
165  char32 codebase = static_cast<char32>(base * kIndicCodePageSize);
166  // Check for validity.
167  if (codebase == static_cast<char32>(ViramaScript::kMyanmar) ||
168  codebase == static_cast<char32>(ViramaScript::kJavanese) ||
169  codebase == static_cast<char32>(ViramaScript::kKhmer) ||
170  (static_cast<char32>(ViramaScript::kDevanagari) <= codebase &&
171  codebase <= static_cast<char32>(ViramaScript::kSinhala))) {
172  return static_cast<ViramaScript>(codebase);
173  }
174  }
176 }
static const char32 kMinIndicUnicode
Definition: validator.h:215
static const char32 kMaxJavaneseUnicode
Definition: validator.h:226
static const int kIndicCodePageSize
Definition: validator.h:213
signed int char32

◆ MoveResultsToDest()

void tesseract::Validator::MoveResultsToDest ( GraphemeNormMode  g_mode,
std::vector< std::vector< char32 >> *  dest 
)
protected

Definition at line 114 of file validator.cpp.

115  {
117  // Append each element of the combined output_ that we made as a new vector
118  // in dest.
119  dest->reserve(dest->size() + output_.size());
120  for (char32 ch : output_) dest->push_back({ch});
121  } else if (g_mode == GraphemeNormMode::kGlyphSplit) {
122  // Append all the parts_ that we made onto dest.
123  std::move(parts_.begin(), parts_.end(), std::back_inserter(*dest));
124  } else if (g_mode == GraphemeNormMode::kCombined || dest->empty()) {
125  // Append the combined output_ that we made onto dest as one new vector.
126  dest->push_back(std::vector<char32>());
127  output_.swap(dest->back());
128  } else { // kNone.
129  // Append the combined output_ that we made onto the last existing element
130  // of dest.
131  dest->back().insert(dest->back().end(), output_.begin(), output_.end());
132  }
133 }
std::vector< std::vector< char32 > > parts_
Definition: validator.h:233
signed int char32
std::vector< char32 > output_
Definition: validator.h:235

◆ MultiCodePart()

void tesseract::Validator::MultiCodePart ( unsigned  length)
inlineprotected

Definition at line 181 of file validator.h.

181  {
182  while (output_used_ + length < output_.size()) {
183  parts_.emplace_back(
184  std::initializer_list<char32>{output_[output_used_++]});
185  }
186  parts_.emplace_back(std::initializer_list<char32>{output_[output_used_]});
187  while (++output_used_ < output_.size()) {
188  parts_.back().push_back(output_[output_used_]);
189  }
190  }
std::vector< std::vector< char32 > > parts_
Definition: validator.h:233
unsigned output_used_
Definition: validator.h:239
std::vector< char32 > output_
Definition: validator.h:235

◆ ScriptValidator()

std::unique_ptr< Validator > tesseract::Validator::ScriptValidator ( ViramaScript  script,
bool  report_errors 
)
staticprotected

Definition at line 71 of file validator.cpp.

72  {
73  switch (script) {
75  return std::unique_ptr<Validator>(
76  new ValidateGrapheme(script, report_errors));
78  return std::unique_ptr<Validator>(
79  new ValidateJavanese(script, report_errors));
81  return std::unique_ptr<Validator>(
82  new ValidateMyanmar(script, report_errors));
84  return std::unique_ptr<Validator>(
85  new ValidateKhmer(script, report_errors));
86  default:
87  return std::unique_ptr<Validator>(
88  new ValidateIndic(script, report_errors));
89  }
90 }

◆ UnicodeToCharClass()

virtual CharClass tesseract::Validator::UnicodeToCharClass ( char32  ch) const
protectedpure virtual

◆ UseMultiCode()

bool tesseract::Validator::UseMultiCode ( unsigned  length)
inlineprotected

Definition at line 195 of file validator.h.

195  {
196  output_.push_back(codes_[codes_used_].second);
197  MultiCodePart(length);
198  return ++codes_used_ == codes_.size();
199  }
void MultiCodePart(unsigned length)
Definition: validator.h:181
unsigned codes_used_
Definition: validator.h:237
std::vector< char32 > output_
Definition: validator.h:235
std::vector< IndicPair > codes_
Definition: validator.h:231

◆ ValidateCleanAndSegment()

bool tesseract::Validator::ValidateCleanAndSegment ( GraphemeNormMode  g_mode,
bool  report_errors,
const std::vector< char32 > &  src,
std::vector< std::vector< char32 >> *  dest 
)
static

Definition at line 40 of file validator.cpp.

42  {
43  ValidateGrapheme g_validator(ViramaScript::kNonVirama, report_errors);
44  std::vector<std::vector<char32>> graphemes;
46  bool success = true;
47  if (script == ViramaScript::kNonVirama) {
48  // The grapheme segmenter's maximum segmentation is the grapheme unit, so
49  // up the mode by 1 to get the desired effect.
50  if (g_mode == GraphemeNormMode::kCombined)
52  else if (g_mode == GraphemeNormMode::kGlyphSplit)
54  // Just do grapheme segmentation.
55  success = g_validator.ValidateCleanAndSegmentInternal(g_mode, src, dest);
56  } else {
57  success = g_validator.ValidateCleanAndSegmentInternal(
58  GraphemeNormMode::kGlyphSplit, src, &graphemes);
59  std::unique_ptr<Validator> validator(
60  ScriptValidator(script, report_errors));
61  for (const auto& grapheme : graphemes) {
62  if (!validator->ValidateCleanAndSegmentInternal(g_mode, grapheme, dest)) {
63  success = false;
64  }
65  }
66  }
67  return success;
68 }
static ViramaScript MostFrequentViramaScript(const std::vector< char32 > &utf32)
Definition: validator.cpp:143
static std::unique_ptr< Validator > ScriptValidator(ViramaScript script, bool report_errors)
Definition: validator.cpp:71

◆ ValidateCleanAndSegmentInternal()

bool tesseract::Validator::ValidateCleanAndSegmentInternal ( GraphemeNormMode  g_mode,
const std::vector< char32 > &  src,
std::vector< std::vector< char32 >> *  dest 
)
protected

Definition at line 97 of file validator.cpp.

99  {
100  Clear();
101  ComputeClassCodes(src);
102  bool success = true;
103  for (codes_used_ = 0; codes_used_ < codes_.size();) {
104  if (!ConsumeGraphemeIfValid()) {
105  success = false;
106  ++codes_used_;
107  }
108  }
109  MoveResultsToDest(g_mode, dest);
110  return success;
111 }
unsigned codes_used_
Definition: validator.h:237
void ComputeClassCodes(const std::vector< char32 > &text)
Definition: validator.cpp:206
std::vector< IndicPair > codes_
Definition: validator.h:231
void MoveResultsToDest(GraphemeNormMode g_mode, std::vector< std::vector< char32 >> *dest)
Definition: validator.cpp:114
virtual bool ConsumeGraphemeIfValid()=0

Member Data Documentation

◆ codes_

std::vector<IndicPair> tesseract::Validator::codes_
protected

Definition at line 231 of file validator.h.

◆ codes_used_

unsigned tesseract::Validator::codes_used_
protected

Definition at line 237 of file validator.h.

◆ kIndicCodePageSize

const int tesseract::Validator::kIndicCodePageSize = 128
staticprotected

Definition at line 213 of file validator.h.

◆ kInvalid

const char32 tesseract::Validator::kInvalid = 0xfffd
static

Definition at line 99 of file validator.h.

◆ kJavaneseVirama

const char32 tesseract::Validator::kJavaneseVirama = 0xa9c0
staticprotected

Definition at line 225 of file validator.h.

◆ kKhmerVirama

const char32 tesseract::Validator::kKhmerVirama = 0x17d2
staticprotected

Definition at line 223 of file validator.h.

◆ kLeftToRightMark

const char32 tesseract::Validator::kLeftToRightMark = 0x200E
static

Definition at line 97 of file validator.h.

◆ kMaxJavaneseUnicode

const char32 tesseract::Validator::kMaxJavaneseUnicode = 0xa9df
staticprotected

Definition at line 226 of file validator.h.

◆ kMaxSinhalaUnicode

const char32 tesseract::Validator::kMaxSinhalaUnicode = 0xdff
staticprotected

Definition at line 217 of file validator.h.

◆ kMaxViramaScriptUnicode

const char32 tesseract::Validator::kMaxViramaScriptUnicode = 0x17ff
staticprotected

Definition at line 219 of file validator.h.

◆ kMinIndicUnicode

const char32 tesseract::Validator::kMinIndicUnicode = 0x900
staticprotected

Definition at line 215 of file validator.h.

◆ kMyanmarVirama

const char32 tesseract::Validator::kMyanmarVirama = 0x1039
staticprotected

Definition at line 222 of file validator.h.

◆ kRightToLeftMark

const char32 tesseract::Validator::kRightToLeftMark = 0x200F
static

Definition at line 98 of file validator.h.

◆ kSinhalaVirama

const char32 tesseract::Validator::kSinhalaVirama = 0xdca
staticprotected

Definition at line 221 of file validator.h.

◆ kZeroWidthJoiner

const char32 tesseract::Validator::kZeroWidthJoiner = 0x200D
static

Definition at line 96 of file validator.h.

◆ kZeroWidthNonJoiner

const char32 tesseract::Validator::kZeroWidthNonJoiner = 0x200C
static

Definition at line 95 of file validator.h.

◆ kZeroWidthSpace

const char32 tesseract::Validator::kZeroWidthSpace = 0x200B
static

Definition at line 94 of file validator.h.

◆ output_

std::vector<char32> tesseract::Validator::output_
protected

Definition at line 235 of file validator.h.

◆ output_used_

unsigned tesseract::Validator::output_used_
protected

Definition at line 239 of file validator.h.

◆ parts_

std::vector<std::vector<char32> > tesseract::Validator::parts_
protected

Definition at line 233 of file validator.h.

◆ report_errors_

bool tesseract::Validator::report_errors_
protected

Definition at line 241 of file validator.h.

◆ script_

ViramaScript tesseract::Validator::script_
protected

Definition at line 229 of file validator.h.


The documentation for this class was generated from the following files: