gpt-tokenizer
v1.0.5 BPE encoder decoder tokenizer GPT GPT-2 GPT-3 GPT-3.5 GPT-4 and 7 more...
test
yarn test:format && yarn test:types && yarn test:lint && yarn test:code
build
yarn build:cjs && yarn build:esm && yarn build:umd
clean
git clean -dfX --exclude=node_modules src && beemo typescript:sync-project-refs
format
yarn rrun prettier --write "./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}"
prepare
rrun husky install .config/husky && beemo create-config
release
beemo run-script release
build:cjs
yarn rrun tsc --outDir cjs --module commonjs --target es2022
build:esm
yarn rrun tsc --outDir esm --module esnext --target es2022 && echo '{"name": "gpt-tokenizer", "type": "module"}' > ./esm/package.json
build:umd
beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPT3Encoder' --env 'export=default' --env 'filename=gpt3encoder.js'
test:code
rrun jest
test:lint
rrun eslint 'src/*.{js,jsx,ts,tsx}'
test:types
yarn rrun tsc --noEmit
test:format
yarn rrun prettier --check "./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}"
postinstallDev
yarn prepare
npm i [email protected]
Metadata
- MIT
- Whatever
- Bazyli Brzoska
- released 5/16/2023