gpt-tokenizer

v1.0.0
BPE Encoder Decoder for GPT-2 / GPT-3
BPE encoder decoder tokenizer GPT GPT-2 GPT-3 GPT-3.5 GPT-4 and 7 more...

Readme Versions Dependencies Scripts

test

yarn test:format && yarn test:types && yarn test:lint && yarn test:code

build

yarn build:cjs && yarn build:esm && yarn build:umd

clean

git clean -dfX --exclude=node_modules src && beemo typescript:sync-project-refs

format

yarn rrun prettier --write "./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}"

prepare

rrun husky install .config/husky && beemo create-config

release

beemo run-script release

build:cjs

yarn rrun tsc --outDir cjs --module commonjs --target es2022

build:esm

yarn rrun tsc --outDir esm --module esnext --target es2022

build:umd

beemo webpack --entry='./src/main.ts' --env 'outDir=dist' --env 'moduleTarget=umd' --env 'engineTarget=web' --env 'codeTarget=es2022' --env 'name=GPT3Encoder' --env 'export=default' --env 'filename=gpt3encoder.js'

test:code

rrun jest

test:lint

rrun eslint 'src/**/*.{js,jsx,ts,tsx}'

test:types

yarn rrun tsc --noEmit

test:format

yarn rrun prettier --check "./{src,tests,.config}/**/!(*.d).{.js,jsx,ts,tsx,json,md}"

postinstallDev

yarn prepare
npm i [email protected]

Metadata

  • MIT
  • Whatever
  • Bazyli Brzoska
  • released 4/16/2023

Downloads

Maintainers