Serialized Form


Package edu.berkeley.nlp.lm

Class edu.berkeley.nlp.lm.AbstractArrayEncodedNgramLanguageModel extends AbstractNgramLanguageModel<W> implements Serializable

serialVersionUID: 1L

Class edu.berkeley.nlp.lm.AbstractContextEncodedNgramLanguageModel extends AbstractNgramLanguageModel<W> implements Serializable

serialVersionUID: 1L

Class edu.berkeley.nlp.lm.AbstractNgramLanguageModel extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

lmOrder

int lmOrder

wordIndexer

WordIndexer<W> wordIndexer

oovWordLogProb

float oovWordLogProb
Fixed constant returned when computing the log probability for an n-gram whose last word is not in the vocabulary. Note that this is different from the log prob of the unk tag probability.

Class edu.berkeley.nlp.lm.ArrayEncodedProbBackoffLm extends AbstractArrayEncodedNgramLanguageModel<W> implements Serializable

serialVersionUID: 1L

Serialized Fields

map

NgramMap<V> map

values

ProbBackoffValueContainer values

useScratchValues

boolean useScratchValues

numWords

long numWords

Class edu.berkeley.nlp.lm.ConfigOptions extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

compressedBlockSize

int compressedBlockSize

offsetDeltaRadix

int offsetDeltaRadix

valueRadix

int valueRadix

hashTableLoadFactor

double hashTableLoadFactor

unknownWordLogProb

double unknownWordLogProb

stupidBackoffAlpha

double stupidBackoffAlpha

kneserNeyDiscounts

double[] kneserNeyDiscounts

kneserNeyMinCounts

double[] kneserNeyMinCounts

numWordBits

int numWordBits

lockIndexer

boolean lockIndexer

storeRankedProbBackoffs

boolean storeRankedProbBackoffs

Class edu.berkeley.nlp.lm.ContextEncodedProbBackoffLm extends AbstractContextEncodedNgramLanguageModel<W> implements Serializable

serialVersionUID: 1L

Serialized Fields

map

HashNgramMap<T> map

values

ProbBackoffValueContainer values

numWords

long numWords

Class edu.berkeley.nlp.lm.StringWordIndexer extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

sparseIndexer

Indexer<E extends Comparable<E>> sparseIndexer

startSymbol

String startSymbol

endSymbol

String endSymbol

unkSymbol

String unkSymbol

unkIndex

int unkIndex

Class edu.berkeley.nlp.lm.StupidBackoffLm extends AbstractArrayEncodedNgramLanguageModel<W> implements Serializable

serialVersionUID: 1L

Serialized Fields

map

NgramMap<V> map

alpha

float alpha

Package edu.berkeley.nlp.lm.array

Class edu.berkeley.nlp.lm.array.CustomWidthArray extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

size

long size

keyWidth

int keyWidth

fullWidth

int fullWidth

widthDiff

long widthDiff

data

LongArray data

Class edu.berkeley.nlp.lm.array.LongArray extends Object implements Serializable

serialVersionUID: -9133624434714616987L

Serialized Fields

size

long size

data

long[][] data

first

long[] first

Package edu.berkeley.nlp.lm.bits

Class edu.berkeley.nlp.lm.bits.VariableLengthBitCompressor extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

radix

int radix

Package edu.berkeley.nlp.lm.cache

Class edu.berkeley.nlp.lm.cache.ArrayEncodedCachingLmWrapper extends AbstractArrayEncodedNgramLanguageModel<W> implements Serializable

serialVersionUID: 1L

Serialized Fields

cache

ArrayEncodedLmCache cache

lm

ArrayEncodedNgramLanguageModel<W> lm

capacity

int capacity

Class edu.berkeley.nlp.lm.cache.ArrayEncodedDirectMappedLmCache extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

threadUnsafeArray

int[] threadUnsafeArray

threadSafeArray

ThreadLocal<T> threadSafeArray

cacheSize

int cacheSize

structLength

int structLength

threadSafe

boolean threadSafe

arrayLength

int arrayLength

Class edu.berkeley.nlp.lm.cache.ContextEncodedCachingLmWrapper extends AbstractContextEncodedNgramLanguageModel<T> implements Serializable

serialVersionUID: 1L

Serialized Fields

contextCache

ContextEncodedLmCache contextCache

lm

ContextEncodedNgramLanguageModel<W> lm

capacity

int capacity

Class edu.berkeley.nlp.lm.cache.ContextEncodedDirectMappedLmCache extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

threadUnsafeArray

long[] threadUnsafeArray

threadSafeArray

ThreadLocal<T> threadSafeArray

cacheSize

int cacheSize

threadSafe

boolean threadSafe

Package edu.berkeley.nlp.lm.collections

Class edu.berkeley.nlp.lm.collections.AbstractTMap extends Object implements Serializable

serialVersionUID: 42L

Serialized Fields

mapType

AbstractTMap.MapType mapType

locked

boolean locked

num

int num

keys

Comparable<T>[] keys

keyFunc

AbstractTMap.Functionality<T extends Comparable> keyFunc

numCollisions

int numCollisions

Class edu.berkeley.nlp.lm.collections.AbstractTMap.Functionality extends Object implements Serializable

serialVersionUID: 1L

Class edu.berkeley.nlp.lm.collections.Counter extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

entries

Map<K,V> entries

dirty

boolean dirty

cacheTotal

double cacheTotal

defaultCount

double defaultCount

Class edu.berkeley.nlp.lm.collections.Indexer extends Object implements Serializable

serialVersionUID: -8769544079136550516L

Serialized Fields

objects

ArrayList<E> objects

indexes

TIntMap<T extends Comparable> indexes

locked

boolean locked

sem

Semaphore sem

Class edu.berkeley.nlp.lm.collections.LongHashSet extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

keys

long[] keys

size

int size

maxLoadFactor

double maxLoadFactor

Class edu.berkeley.nlp.lm.collections.TIntMap extends AbstractTMap<T extends Comparable> implements Serializable

serialVersionUID: 42L

Serialized Fields

values

int[] values

Package edu.berkeley.nlp.lm.io

Class edu.berkeley.nlp.lm.io.KneserNeyLmReaderCallback extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

lmOrder

int lmOrder

wordIndexer

WordIndexer<W> wordIndexer
This array represents the discount used for each ngram order. The original Kneser-Ney discounting (-ukndiscount) uses one discounting constant for each N-gram order. These constants are estimated as D = n1 / (n1 + 2*n2) where n1 and n2 are the total number of N-grams with exactly one and two counts, respectively. For simplicity, our code just uses a constant discount for each order of 0.75. However, other discounts can be specified.


ngrams

HashNgramMap<T> ngrams

opts

ConfigOptions opts

startIndex

int startIndex

Package edu.berkeley.nlp.lm.map

Class edu.berkeley.nlp.lm.map.AbstractNgramMap extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

NUM_WORD_BITS

int NUM_WORD_BITS

NUM_SUFFIX_BITS

int NUM_SUFFIX_BITS

WORD_BIT_MASK

long WORD_BIT_MASK

SUFFIX_BIT_MASK

long SUFFIX_BIT_MASK

values

ValueContainer<V> values

opts

ConfigOptions opts

Class edu.berkeley.nlp.lm.map.CompressedNgramMap extends AbstractNgramMap<T> implements Serializable

serialVersionUID: 1L

Serialized Fields

compressedBlockSize

int compressedBlockSize

offsetCoder

VariableLengthBitCompressor offsetCoder

wordCoder

VariableLengthBitCompressor wordCoder

suffixCoder

VariableLengthBitCompressor suffixCoder

totalKeyBitsFinal

double totalKeyBitsFinal

totalValueBitsFinal

double totalValueBitsFinal

totalBitsFinal

double totalBitsFinal

totalSizeFinal

double totalSizeFinal

offsetDeltaRadix

int offsetDeltaRadix

maps

edu.berkeley.nlp.lm.map.CompressedMap[] maps

reverseTrie

boolean reverseTrie

numNgramsForEachOrder

long[] numNgramsForEachOrder

Class edu.berkeley.nlp.lm.map.HashNgramMap extends AbstractNgramMap<T> implements Serializable

serialVersionUID: 1L

Serialized Fields

explicitMaps

edu.berkeley.nlp.lm.map.ExplicitWordHashMap[] explicitMaps

implicitMaps

edu.berkeley.nlp.lm.map.ImplicitWordHashMap[] implicitMaps

implicitUnigramMap

edu.berkeley.nlp.lm.map.UnigramHashMap implicitUnigramMap

initCapacities

long[] initCapacities

maxLoadFactor

double maxLoadFactor

isExplicit

boolean isExplicit

reversed

boolean reversed

storeSuffixOffsets

boolean storeSuffixOffsets

Package edu.berkeley.nlp.lm.phrasetable

Class edu.berkeley.nlp.lm.phrasetable.MosesPhraseTable extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

map

HashNgramMap<T> map

wordIndexer

WordIndexer<W> wordIndexer

Class edu.berkeley.nlp.lm.phrasetable.PhraseTableValueContainer extends Object implements Serializable

serialVersionUID: 964277160049236607L

Serialized Fields

features

LongArray[] features

valueIndexes

LongArray[] valueIndexes

targetTranslations

ArrayList<E>[] targetTranslations

map

HashNgramMap<T> map

separatorWord

int separatorWord

numFeatures

int numFeatures

Class edu.berkeley.nlp.lm.phrasetable.PhraseTableValueContainer.FeaturePhraseTableValues extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

features

float[] features

Class edu.berkeley.nlp.lm.phrasetable.PhraseTableValueContainer.TargetTranslationsValues extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

targetTranslationOffsets

long[] targetTranslationOffsets

targetTranslationOrders

int[] targetTranslationOrders

Package edu.berkeley.nlp.lm.util

Class edu.berkeley.nlp.lm.util.LongRef extends Object implements Serializable

serialVersionUID: 1L

Serialized Fields

value

long value

Class edu.berkeley.nlp.lm.util.Pair extends Object implements Serializable

serialVersionUID: 42L

Serialized Fields

first

Object first

second

Object second

Package edu.berkeley.nlp.lm.values

Class edu.berkeley.nlp.lm.values.CompressibleProbBackoffValueContainer extends edu.berkeley.nlp.lm.values.RankedValueContainer<ProbBackoffPair> implements Serializable

serialVersionUID: 964277160049236607L

Serialized Fields

backoffsForRank

float[] backoffsForRank

probsForRank

float[] probsForRank

backoffWidth

int backoffWidth

Class edu.berkeley.nlp.lm.values.CountValueContainer extends edu.berkeley.nlp.lm.values.RankedValueContainer<LongRef> implements Serializable

serialVersionUID: 964277160049236607L

Serialized Fields

countsForRank

long[] countsForRank

unigramSum

long unigramSum

Class edu.berkeley.nlp.lm.values.KneserNeyCountValueContainer extends Object implements Serializable

serialVersionUID: 964277160049236607L

Serialized Fields

tokenCounts

LongArray tokenCounts

prefixTokenCounts

LongArray prefixTokenCounts

rightDotTypeCounts

LongArray[] rightDotTypeCounts

dotdotTypeCounts

LongArray[] dotdotTypeCounts

leftDotTypeCounts

LongArray[] leftDotTypeCounts

oneCountOffsets

LongHashSet[] oneCountOffsets

twoCountOffsets

LongHashSet[] twoCountOffsets

bigramTypeCounts

long bigramTypeCounts

map

HashNgramMap<T> map

startIndex

int startIndex

Class edu.berkeley.nlp.lm.values.UncompressedProbBackoffValueContainer extends edu.berkeley.nlp.lm.values.RankedValueContainer<ProbBackoffPair> implements Serializable

serialVersionUID: 964277160049236607L

Serialized Fields

probsAndBackoffsForRank

long[] probsAndBackoffsForRank

Class edu.berkeley.nlp.lm.values.UnrankedUncompressedProbBackoffValueContainer extends Object implements Serializable

serialVersionUID: 964277160049236607L

Serialized Fields

storeSuffixIndexes

boolean storeSuffixIndexes

suffixBitsForOrder

int[] suffixBitsForOrder

numNgramsForEachOrder

long[] numNgramsForEachOrder

valueRanks

CustomWidthArray[] valueRanks

ngramMap

NgramMap<V> ngramMap