Serialized Form
-
Package edu.berkeley.nlp.lm
-
Class edu.berkeley.nlp.lm.AbstractArrayEncodedNgramLanguageModel
class AbstractArrayEncodedNgramLanguageModel extends AbstractNgramLanguageModel<W> implements Serializable- serialVersionUID:
- 1L
-
Class edu.berkeley.nlp.lm.AbstractContextEncodedNgramLanguageModel
class AbstractContextEncodedNgramLanguageModel extends AbstractNgramLanguageModel<W> implements Serializable- serialVersionUID:
- 1L
-
Class edu.berkeley.nlp.lm.AbstractNgramLanguageModel
class AbstractNgramLanguageModel extends Object implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
lmOrder
int lmOrder
-
oovWordLogProb
float oovWordLogProb
Fixed constant returned when computing the log probability for an n-gram whose last word is not in the vocabulary. Note that this is different from the log prob of theunk
tag probability. -
wordIndexer
WordIndexer<W> wordIndexer
-
-
Class edu.berkeley.nlp.lm.ArrayEncodedProbBackoffLm
class ArrayEncodedProbBackoffLm extends AbstractArrayEncodedNgramLanguageModel<W> implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
map
NgramMap<ProbBackoffPair> map
-
numWords
long numWords
-
useScratchValues
boolean useScratchValues
-
values
ProbBackoffValueContainer values
-
-
Class edu.berkeley.nlp.lm.ConfigOptions
class ConfigOptions extends Object implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
compressedBlockSize
int compressedBlockSize
-
hashTableLoadFactor
double hashTableLoadFactor
-
kneserNeyDiscounts
double[] kneserNeyDiscounts
-
kneserNeyMinCounts
double[] kneserNeyMinCounts
-
lockIndexer
boolean lockIndexer
-
numWordBits
int numWordBits
-
offsetDeltaRadix
int offsetDeltaRadix
-
storeRankedProbBackoffs
boolean storeRankedProbBackoffs
-
stupidBackoffAlpha
double stupidBackoffAlpha
-
unknownWordLogProb
double unknownWordLogProb
-
valueRadix
int valueRadix
-
-
Class edu.berkeley.nlp.lm.ContextEncodedProbBackoffLm
class ContextEncodedProbBackoffLm extends AbstractContextEncodedNgramLanguageModel<W> implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
map
HashNgramMap<ProbBackoffPair> map
-
numWords
long numWords
-
values
ProbBackoffValueContainer values
-
-
Class edu.berkeley.nlp.lm.StringWordIndexer
class StringWordIndexer extends Object implements Serializable- serialVersionUID:
- 1L
-
Class edu.berkeley.nlp.lm.StupidBackoffLm
- serialVersionUID:
- 1L
-
-
Package edu.berkeley.nlp.lm.array
-
Class edu.berkeley.nlp.lm.array.CustomWidthArray
class CustomWidthArray extends Object implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
data
LongArray data
-
fullWidth
int fullWidth
-
keyWidth
int keyWidth
-
size
long size
-
widthDiff
long widthDiff
-
-
Class edu.berkeley.nlp.lm.array.LongArray
class LongArray extends Object implements Serializable- serialVersionUID:
- -9133624434714616987L
-
Serialized Fields
-
data
long[][] data
-
first
long[] first
-
size
long size
-
-
-
Package edu.berkeley.nlp.lm.bits
-
Class edu.berkeley.nlp.lm.bits.VariableLengthBitCompressor
class VariableLengthBitCompressor extends Object implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
radix
int radix
-
-
-
Package edu.berkeley.nlp.lm.cache
-
Class edu.berkeley.nlp.lm.cache.ArrayEncodedCachingLmWrapper
class ArrayEncodedCachingLmWrapper extends AbstractArrayEncodedNgramLanguageModel<W> implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
cache
ArrayEncodedLmCache cache
-
capacity
int capacity
-
lm
ArrayEncodedNgramLanguageModel<W> lm
-
-
Class edu.berkeley.nlp.lm.cache.ArrayEncodedDirectMappedLmCache
class ArrayEncodedDirectMappedLmCache extends Object implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
arrayLength
int arrayLength
-
cacheSize
int cacheSize
-
structLength
int structLength
-
threadSafe
boolean threadSafe
-
threadSafeArray
ThreadLocal<int[]> threadSafeArray
-
threadUnsafeArray
int[] threadUnsafeArray
-
-
Class edu.berkeley.nlp.lm.cache.ContextEncodedCachingLmWrapper
class ContextEncodedCachingLmWrapper extends AbstractContextEncodedNgramLanguageModel<T> implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
capacity
int capacity
-
contextCache
ContextEncodedLmCache contextCache
-
lm
ContextEncodedNgramLanguageModel<T> lm
-
-
Class edu.berkeley.nlp.lm.cache.ContextEncodedDirectMappedLmCache
class ContextEncodedDirectMappedLmCache extends Object implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
cacheSize
int cacheSize
-
threadSafe
boolean threadSafe
-
threadSafeArray
ThreadLocal<long[]> threadSafeArray
-
threadUnsafeArray
long[] threadUnsafeArray
-
-
-
Package edu.berkeley.nlp.lm.collections
-
Class edu.berkeley.nlp.lm.collections.AbstractTMap
class AbstractTMap extends Object implements Serializable- serialVersionUID:
- 42L
-
Serialized Fields
-
keyFunc
AbstractTMap.Functionality<T extends Comparable> keyFunc
-
keys
T extends Comparable[] keys
-
locked
boolean locked
-
mapType
AbstractTMap.MapType mapType
-
num
int num
-
numCollisions
int numCollisions
-
-
Class edu.berkeley.nlp.lm.collections.AbstractTMap.Functionality
class Functionality extends Object implements Serializable- serialVersionUID:
- 1L
-
Class edu.berkeley.nlp.lm.collections.Counter
class Counter extends Object implements Serializable- serialVersionUID:
- 1L
-
Class edu.berkeley.nlp.lm.collections.Indexer
class Indexer extends Object implements Serializable- serialVersionUID:
- -8769544079136550516L
-
Serialized Fields
-
indexes
TIntMap<E extends Comparable<E>> indexes
-
locked
boolean locked
-
objects
ArrayList<E extends Comparable<E>> objects
-
sem
Semaphore sem
-
-
Class edu.berkeley.nlp.lm.collections.LongHashSet
class LongHashSet extends Object implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
keys
long[] keys
-
maxLoadFactor
double maxLoadFactor
-
size
int size
-
-
Class edu.berkeley.nlp.lm.collections.TIntMap
- serialVersionUID:
- 42L
-
Serialized Fields
-
values
int[] values
-
-
-
Package edu.berkeley.nlp.lm.io
-
Class edu.berkeley.nlp.lm.io.KneserNeyLmReaderCallback
class KneserNeyLmReaderCallback extends Object implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
lmOrder
int lmOrder
-
ngrams
HashNgramMap<KneserNeyCountValueContainer.KneserNeyCounts> ngrams
-
opts
ConfigOptions opts
-
startIndex
int startIndex
-
wordIndexer
WordIndexer<W> wordIndexer
This array represents the discount used for each ngram order. The original Kneser-Ney discounting (-ukndiscount) uses one discounting constant for each N-gram order. These constants are estimated as D = n1 / (n1 + 2*n2) where n1 and n2 are the total number of N-grams with exactly one and two counts, respectively. For simplicity, our code just uses a constant discount for each order of 0.75. However, other discounts can be specified.
-
-
-
Package edu.berkeley.nlp.lm.map
-
Class edu.berkeley.nlp.lm.map.AbstractNgramMap
class AbstractNgramMap extends Object implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
NUM_SUFFIX_BITS
int NUM_SUFFIX_BITS
-
NUM_WORD_BITS
int NUM_WORD_BITS
-
opts
ConfigOptions opts
-
SUFFIX_BIT_MASK
long SUFFIX_BIT_MASK
-
values
ValueContainer<T> values
-
WORD_BIT_MASK
long WORD_BIT_MASK
-
-
Class edu.berkeley.nlp.lm.map.CompressedNgramMap
- serialVersionUID:
- 1L
-
Serialized Fields
-
compressedBlockSize
int compressedBlockSize
-
maps
edu.berkeley.nlp.lm.map.CompressedMap[] maps
-
numNgramsForEachOrder
long[] numNgramsForEachOrder
-
offsetCoder
VariableLengthBitCompressor offsetCoder
-
offsetDeltaRadix
int offsetDeltaRadix
-
reverseTrie
boolean reverseTrie
-
suffixCoder
VariableLengthBitCompressor suffixCoder
-
totalBitsFinal
double totalBitsFinal
-
totalKeyBitsFinal
double totalKeyBitsFinal
-
totalSizeFinal
double totalSizeFinal
-
totalValueBitsFinal
double totalValueBitsFinal
-
wordCoder
VariableLengthBitCompressor wordCoder
-
-
Class edu.berkeley.nlp.lm.map.HashNgramMap
- serialVersionUID:
- 1L
-
Serialized Fields
-
explicitMaps
edu.berkeley.nlp.lm.map.ExplicitWordHashMap[] explicitMaps
-
implicitMaps
edu.berkeley.nlp.lm.map.ImplicitWordHashMap[] implicitMaps
-
implicitUnigramMap
edu.berkeley.nlp.lm.map.UnigramHashMap implicitUnigramMap
-
initCapacities
long[] initCapacities
-
isExplicit
boolean isExplicit
-
maxLoadFactor
double maxLoadFactor
-
reversed
boolean reversed
-
storeSuffixOffsets
boolean storeSuffixOffsets
-
-
-
Package edu.berkeley.nlp.lm.phrasetable
-
Class edu.berkeley.nlp.lm.phrasetable.MosesPhraseTable
class MosesPhraseTable extends Object implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
map
HashNgramMap<PhraseTableValueContainer.PhraseTableValues> map
-
wordIndexer
WordIndexer<String> wordIndexer
-
-
Class edu.berkeley.nlp.lm.phrasetable.PhraseTableValueContainer
class PhraseTableValueContainer extends Object implements Serializable- serialVersionUID:
- 964277160049236607L
-
Serialized Fields
-
features
LongArray[] features
-
map
HashNgramMap<PhraseTableValueContainer.PhraseTableValues> map
-
numFeatures
int numFeatures
-
separatorWord
int separatorWord
-
targetTranslations
ArrayList<CustomWidthArray>[] targetTranslations
-
valueIndexes
LongArray[] valueIndexes
-
-
Class edu.berkeley.nlp.lm.phrasetable.PhraseTableValueContainer.FeaturePhraseTableValues
class FeaturePhraseTableValues extends Object implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
features
float[] features
-
-
Class edu.berkeley.nlp.lm.phrasetable.PhraseTableValueContainer.TargetTranslationsValues
class TargetTranslationsValues extends Object implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
targetTranslationOffsets
long[] targetTranslationOffsets
-
targetTranslationOrders
int[] targetTranslationOrders
-
-
-
Package edu.berkeley.nlp.lm.util
-
Class edu.berkeley.nlp.lm.util.LongRef
class LongRef extends Object implements Serializable- serialVersionUID:
- 1L
-
Serialized Fields
-
value
long value
-
-
Class edu.berkeley.nlp.lm.util.Pair
class Pair extends Object implements Serializable- serialVersionUID:
- 42L
-
-
Package edu.berkeley.nlp.lm.values
-
Class edu.berkeley.nlp.lm.values.CompressibleProbBackoffValueContainer
class CompressibleProbBackoffValueContainer extends edu.berkeley.nlp.lm.values.RankedValueContainer<ProbBackoffPair> implements Serializable- serialVersionUID:
- 964277160049236607L
-
Serialized Fields
-
backoffsForRank
float[] backoffsForRank
-
backoffWidth
int backoffWidth
-
probsForRank
float[] probsForRank
-
-
Class edu.berkeley.nlp.lm.values.CountValueContainer
class CountValueContainer extends edu.berkeley.nlp.lm.values.RankedValueContainer<LongRef> implements Serializable- serialVersionUID:
- 964277160049236607L
-
Serialized Fields
-
countsForRank
long[] countsForRank
-
unigramSum
long unigramSum
-
-
Class edu.berkeley.nlp.lm.values.KneserNeyCountValueContainer
class KneserNeyCountValueContainer extends Object implements Serializable- serialVersionUID:
- 964277160049236607L
-
Serialized Fields
-
bigramTypeCounts
long bigramTypeCounts
-
dotdotTypeCounts
LongArray[] dotdotTypeCounts
-
leftDotTypeCounts
LongArray[] leftDotTypeCounts
-
map
HashNgramMap<KneserNeyCountValueContainer.KneserNeyCounts> map
-
oneCountOffsets
LongHashSet[] oneCountOffsets
-
prefixTokenCounts
LongArray prefixTokenCounts
-
rightDotTypeCounts
LongArray[] rightDotTypeCounts
-
startIndex
int startIndex
-
tokenCounts
LongArray tokenCounts
-
twoCountOffsets
LongHashSet[] twoCountOffsets
-
-
Class edu.berkeley.nlp.lm.values.UncompressedProbBackoffValueContainer
class UncompressedProbBackoffValueContainer extends edu.berkeley.nlp.lm.values.RankedValueContainer<ProbBackoffPair> implements Serializable- serialVersionUID:
- 964277160049236607L
-
Serialized Fields
-
probsAndBackoffsForRank
long[] probsAndBackoffsForRank
-
-
Class edu.berkeley.nlp.lm.values.UnrankedUncompressedProbBackoffValueContainer
class UnrankedUncompressedProbBackoffValueContainer extends Object implements Serializable- serialVersionUID:
- 964277160049236607L
-
Serialized Fields
-
ngramMap
NgramMap<ProbBackoffPair> ngramMap
-
numNgramsForEachOrder
long[] numNgramsForEachOrder
-
storeSuffixIndexes
boolean storeSuffixIndexes
-
suffixBitsForOrder
int[] suffixBitsForOrder
-
valueRanks
CustomWidthArray[] valueRanks
-
-