Memory leak

[hlherrera]

if I have an express controller:

Controller.js

`model: string //path

postMethod(text) {
const textClassifier = new Fasttext(model); // here memory leak!
return textClassifier.predict(text);
}
`
Memory grows without limit on every request.

[vunb]

Hi, you should try to declare textClassifier as a reference variable at top level of controller. Don't create new in every requests.

Example:

Controller.js

model: string //path
textClassifier = new Fasttext(model);

postMethod(text) {
  return textClassifier.predict(text);
}

[alexandrecoulay]

Hello,
I just discovered the package, but if you still use it, here is a way to avoid memory leak by using a Singleton.
Here is my code, I use it to return language of a text for my social network :

import FastText, { Classifier } from 'fasttext';

export type ISO_639_CODE_LIST = 'af' | 'als' | 'am' | 'an' | 'ar' | 'arz' | 'as' | 'ast' | 'av' | 'az' | 'azb' | 'ba' | 'bar' | 'bcl' | 'be' | 'bg' | 'bh' | 'bn' | 'bo' | 'bpy' | 'br' | 'bs' | 'bxr' | 'ca' | 'cbk' | 'ce' | 'ceb' | 'ckb' | 'co' | 'cs' | 'cv' | 'cy' | 'da' | 'de' | 'diq' | 'dsb' | 'dty' | 'dv' | 'el' | 'eml' | 'en' | 'eo' | 'es' | 'et' | 'eu' | 'fa' | 'fi' | 'fr' | 'frr' | 'fy' | 'ga' | 'gd' | 'gl' | 'gn' | 'gom' | 'gu' | 'gv' | 'he' | 'hi' | 'hif' | 'hr' | 'hsb' | 'ht' | 'hu' | 'hy' | 'ia' | 'id' | 'ie' | 'ilo' | 'io' | 'is' | 'it' | 'ja' | 'jbo' | 'jv' | 'ka' | 'kk' | 'km' | 'kn' | 'ko' | 'krc' | 'ku' | 'kv' | 'kw' | 'ky' | 'la' | 'lb' | 'lez' | 'li' | 'lmo' | 'lo' | 'lrc' | 'lt' | 'lv' | 'mai' | 'mg' | 'mhr' | 'min' | 'mk' | 'ml' | 'mn' | 'mr' | 'mrj' | 'ms' | 'mt' | 'mwl' | 'my' | 'myv' | 'mzn' | 'nah' | 'nap' | 'nds' | 'ne' | 'new' | 'nl' | 'nn' | 'no' | 'oc' | 'or' | 'os' | 'pa' | 'pam' | 'pfl' | 'pl' | 'pms' | 'pnb' | 'ps' | 'pt' | 'qu' | 'rm' | 'ro' | 'ru' | 'rue' | 'sa' | 'sah' | 'sc' | 'scn' | 'sco' | 'sd' | 'sh' | 'si' | 'sk' | 'sl' | 'so' | 'sq' | 'sr' | 'su' | 'sv' | 'sw' | 'ta' | 'te' | 'tg' | 'th' | 'tk' | 'tl' | 'tr' | 'tt' | 'tyv' | 'ug' | 'uk' | 'ur' | 'uz' | 'vec' | 'vep' | 'vi' | 'vls' | 'vo' | 'wa' | 'war' | 'wuu' | 'xal' | 'xmf' | 'yi' | 'yo' | 'yue' | 'zh';

// from benchmark-testing/results/reliability_list_<verison>.json
const reliabilityList = [
    'th',
    'ml',
    'my',
    'ta',
    'te',
    'pa',
    'am',
    'kn',
    'gu',
    'si',
    'bo',
    'dv',
    'ja',
    'el',
    'he',
    'ko',
    'hy',
    'bn',
    'mr',
    'en',
    'zh',
    'tr',
    'ru',
    'de',
    'ug',
    'vi',
    'eo',
    'ka',
    'hi',
    'it',
    'ar',
    'fr',
    'hu',
    'lo',
    'pl',
    'km',
    'es',
    'fi',
    'pt',
    'mk',
    'uk',
    'ur',
    'nl',
    'lt',
    'cs',
]

const languageIsoCodes = [
    'af',
    'als',
    'am',
    'an',
    'ar',
    'arz',
    'as',
    'ast',
    'av',
    'az',
    'azb',
    'ba',
    'bar',
    'bcl',
    'be',
    'bg',
    'bh',
    'bn',
    'bo',
    'bpy',
    'br',
    'bs',
    'bxr',
    'ca',
    'cbk',
    'ce',
    'ceb',
    'ckb',
    'co',
    'cs',
    'cv',
    'cy',
    'da',
    'de',
    'diq',
    'dsb',
    'dty',
    'dv',
    'el',
    'eml',
    'en',
    'eo',
    'es',
    'et',
    'eu',
    'fa',
    'fi',
    'fr',
    'frr',
    'fy',
    'ga',
    'gd',
    'gl',
    'gn',
    'gom',
    'gu',
    'gv',
    'he',
    'hi',
    'hif',
    'hr',
    'hsb',
    'ht',
    'hu',
    'hy',
    'ia',
    'id',
    'ie',
    'ilo',
    'io',
    'is',
    'it',
    'ja',
    'jbo',
    'jv',
    'ka',
    'kk',
    'km',
    'kn',
    'ko',
    'krc',
    'ku',
    'kv',
    'kw',
    'ky',
    'la',
    'lb',
    'lez',
    'li',
    'lmo',
    'lo',
    'lrc',
    'lt',
    'lv',
    'mai',
    'mg',
    'mhr',
    'min',
    'mk',
    'ml',
    'mn',
    'mr',
    'mrj',
    'ms',
    'mt',
    'mwl',
    'my',
    'myv',
    'mzn',
    'nah',
    'nap',
    'nds',
    'ne',
    'new',
    'nl',
    'nn',
    'no',
    'oc',
    'or',
    'os',
    'pa',
    'pam',
    'pfl',
    'pl',
    'pms',
    'pnb',
    'ps',
    'pt',
    'qu',
    'rm',
    'ro',
    'ru',
    'rue',
    'sa',
    'sah',
    'sc',
    'scn',
    'sco',
    'sd',
    'sh',
    'si',
    'sk',
    'sl',
    'so',
    'sq',
    'sr',
    'su',
    'sv',
    'sw',
    'ta',
    'te',
    'tg',
    'th',
    'tk',
    'tl',
    'tr',
    'tt',
    'tyv',
    'ug',
    'uk',
    'ur',
    'uz',
    'vec',
    'vep',
    'vi',
    'vls',
    'vo',
    'wa',
    'war',
    'wuu',
    'xal',
    'xmf',
    'yi',
    'yo',
    'yue',
    'zh',
]

// Some characters can misconstrude the results
// Removing \n and : because of this issue: https://github.com/indix/whatthelang/issues/12
function formatInput(text: string) {
    return text.replace(/[\n:]/g, '')
}

class FastTextSingleton {
    private static instance: Classifier;
    private constructor() { }

    public static getInstance(): Classifier {
        if (!FastTextSingleton.instance) return FastTextSingleton.instance = new FastText.Classifier('./assets/models/language-detect/lid.176.bin');
        return FastTextSingleton.instance;
    }
}

/*  the API class  */
export class LanguageDetection {

    public classifier: Classifier;
    public languageIsoCodes: any;

    constructor() {
        this.classifier = FastTextSingleton.getInstance();
        this.languageIsoCodes = languageIsoCodes;
    }
    predict(text: string, k = 5): Promise<Array<{
        lang: ISO_639_CODE_LIST,
        prob: number,
        isReliableLanguage: boolean
    }>> {
        return new Promise((resolve, reject) => {
            this.classifier.predict(formatInput(text), k).then((result: any) => {
                const to_return = result.map((item: any) => {
                    const lang = item.label.replace(/^__label__/, '')
                    return {
                        lang,
                        prob: item.value,
                        isReliableLanguage: reliabilityList.includes(lang),
                    }
                })
                resolve(to_return)
            }).catch((error: any) => {
                reject(error);
            });
        })
    }
}