Serialized Form
Package edu.berkeley.nlp.lm |
serialVersionUID: 1L
serialVersionUID: 1L
serialVersionUID: 1L
lmOrder
int lmOrder
wordIndexer
WordIndexer<W> wordIndexer
oovWordLogProb
float oovWordLogProb
- Fixed constant returned when computing the log probability for an n-gram
whose last word is not in the vocabulary. Note that this is different
from the log prob of the
unk
tag probability.
serialVersionUID: 1L
map
NgramMap<V> map
values
ProbBackoffValueContainer values
useScratchValues
boolean useScratchValues
numWords
long numWords
serialVersionUID: 1L
compressedBlockSize
int compressedBlockSize
offsetDeltaRadix
int offsetDeltaRadix
valueRadix
int valueRadix
hashTableLoadFactor
double hashTableLoadFactor
unknownWordLogProb
double unknownWordLogProb
stupidBackoffAlpha
double stupidBackoffAlpha
kneserNeyDiscounts
double[] kneserNeyDiscounts
kneserNeyMinCounts
double[] kneserNeyMinCounts
numWordBits
int numWordBits
lockIndexer
boolean lockIndexer
storeRankedProbBackoffs
boolean storeRankedProbBackoffs
serialVersionUID: 1L
map
HashNgramMap<T> map
values
ProbBackoffValueContainer values
numWords
long numWords
serialVersionUID: 1L
sparseIndexer
Indexer<E extends Comparable<E>> sparseIndexer
startSymbol
String startSymbol
endSymbol
String endSymbol
unkSymbol
String unkSymbol
unkIndex
int unkIndex
serialVersionUID: 1L
map
NgramMap<V> map
alpha
float alpha
Package edu.berkeley.nlp.lm.array |
serialVersionUID: 1L
size
long size
keyWidth
int keyWidth
fullWidth
int fullWidth
widthDiff
long widthDiff
data
LongArray data
serialVersionUID: -9133624434714616987L
size
long size
data
long[][] data
first
long[] first
Package edu.berkeley.nlp.lm.bits |
serialVersionUID: 1L
radix
int radix
Package edu.berkeley.nlp.lm.cache |
serialVersionUID: 1L
cache
ArrayEncodedLmCache cache
lm
ArrayEncodedNgramLanguageModel<W> lm
capacity
int capacity
serialVersionUID: 1L
threadUnsafeArray
int[] threadUnsafeArray
threadSafeArray
ThreadLocal<T> threadSafeArray
cacheSize
int cacheSize
structLength
int structLength
threadSafe
boolean threadSafe
arrayLength
int arrayLength
serialVersionUID: 1L
contextCache
ContextEncodedLmCache contextCache
lm
ContextEncodedNgramLanguageModel<W> lm
capacity
int capacity
serialVersionUID: 1L
threadUnsafeArray
long[] threadUnsafeArray
threadSafeArray
ThreadLocal<T> threadSafeArray
cacheSize
int cacheSize
threadSafe
boolean threadSafe
Package edu.berkeley.nlp.lm.collections |
serialVersionUID: 42L
mapType
AbstractTMap.MapType mapType
locked
boolean locked
num
int num
keys
Comparable<T>[] keys
keyFunc
AbstractTMap.Functionality<T extends Comparable> keyFunc
numCollisions
int numCollisions
serialVersionUID: 1L
serialVersionUID: 1L
entries
Map<K,V> entries
dirty
boolean dirty
cacheTotal
double cacheTotal
defaultCount
double defaultCount
serialVersionUID: -8769544079136550516L
objects
ArrayList<E> objects
indexes
TIntMap<T extends Comparable> indexes
locked
boolean locked
sem
Semaphore sem
serialVersionUID: 1L
keys
long[] keys
size
int size
maxLoadFactor
double maxLoadFactor
serialVersionUID: 42L
values
int[] values
Package edu.berkeley.nlp.lm.io |
serialVersionUID: 1L
lmOrder
int lmOrder
wordIndexer
WordIndexer<W> wordIndexer
- This array represents the discount used for each ngram order.
The original Kneser-Ney discounting (-ukndiscount) uses one discounting
constant for each N-gram order. These constants are estimated as
D = n1 / (n1 + 2*n2)
where n1 and n2 are the total number of N-grams with exactly one and two
counts, respectively.
For simplicity, our code just uses a constant discount for each order of
0.75. However, other discounts can be specified.
ngrams
HashNgramMap<T> ngrams
opts
ConfigOptions opts
startIndex
int startIndex
Package edu.berkeley.nlp.lm.map |
serialVersionUID: 1L
NUM_WORD_BITS
int NUM_WORD_BITS
NUM_SUFFIX_BITS
int NUM_SUFFIX_BITS
WORD_BIT_MASK
long WORD_BIT_MASK
SUFFIX_BIT_MASK
long SUFFIX_BIT_MASK
values
ValueContainer<V> values
opts
ConfigOptions opts
serialVersionUID: 1L
compressedBlockSize
int compressedBlockSize
offsetCoder
VariableLengthBitCompressor offsetCoder
wordCoder
VariableLengthBitCompressor wordCoder
suffixCoder
VariableLengthBitCompressor suffixCoder
totalKeyBitsFinal
double totalKeyBitsFinal
totalValueBitsFinal
double totalValueBitsFinal
totalBitsFinal
double totalBitsFinal
totalSizeFinal
double totalSizeFinal
offsetDeltaRadix
int offsetDeltaRadix
maps
edu.berkeley.nlp.lm.map.CompressedMap[] maps
reverseTrie
boolean reverseTrie
numNgramsForEachOrder
long[] numNgramsForEachOrder
serialVersionUID: 1L
explicitMaps
edu.berkeley.nlp.lm.map.ExplicitWordHashMap[] explicitMaps
implicitMaps
edu.berkeley.nlp.lm.map.ImplicitWordHashMap[] implicitMaps
implicitUnigramMap
edu.berkeley.nlp.lm.map.UnigramHashMap implicitUnigramMap
initCapacities
long[] initCapacities
maxLoadFactor
double maxLoadFactor
isExplicit
boolean isExplicit
reversed
boolean reversed
storeSuffixOffsets
boolean storeSuffixOffsets
Package edu.berkeley.nlp.lm.phrasetable |
serialVersionUID: 1L
map
HashNgramMap<T> map
wordIndexer
WordIndexer<W> wordIndexer
serialVersionUID: 964277160049236607L
features
LongArray[] features
valueIndexes
LongArray[] valueIndexes
targetTranslations
ArrayList<E>[] targetTranslations
map
HashNgramMap<T> map
separatorWord
int separatorWord
numFeatures
int numFeatures
serialVersionUID: 1L
features
float[] features
serialVersionUID: 1L
targetTranslationOffsets
long[] targetTranslationOffsets
targetTranslationOrders
int[] targetTranslationOrders
Package edu.berkeley.nlp.lm.util |
serialVersionUID: 1L
value
long value
serialVersionUID: 42L
first
Object first
second
Object second
Package edu.berkeley.nlp.lm.values |
serialVersionUID: 964277160049236607L
backoffsForRank
float[] backoffsForRank
probsForRank
float[] probsForRank
backoffWidth
int backoffWidth
serialVersionUID: 964277160049236607L
countsForRank
long[] countsForRank
unigramSum
long unigramSum
serialVersionUID: 964277160049236607L
tokenCounts
LongArray tokenCounts
prefixTokenCounts
LongArray prefixTokenCounts
rightDotTypeCounts
LongArray[] rightDotTypeCounts
dotdotTypeCounts
LongArray[] dotdotTypeCounts
leftDotTypeCounts
LongArray[] leftDotTypeCounts
oneCountOffsets
LongHashSet[] oneCountOffsets
twoCountOffsets
LongHashSet[] twoCountOffsets
bigramTypeCounts
long bigramTypeCounts
map
HashNgramMap<T> map
startIndex
int startIndex
serialVersionUID: 964277160049236607L
probsAndBackoffsForRank
long[] probsAndBackoffsForRank
serialVersionUID: 964277160049236607L
storeSuffixIndexes
boolean storeSuffixIndexes
suffixBitsForOrder
int[] suffixBitsForOrder
numNgramsForEachOrder
long[] numNgramsForEachOrder
valueRanks
CustomWidthArray[] valueRanks
ngramMap
NgramMap<V> ngramMap