revert tokenize ffi (#4761)

* Revert "use `int32_t` for call to tokenize (#4738)"

This reverts commit 763bb65dbb.

* Revert "vocab only"

This reverts commit bf54c845e9.

* Revert "use ffi for tokenizing/detokenizing"

This reverts commit 26a00a0410.

llm/llm.go

@@ -12,7 +12,6 @@ package llm
 import "C"
 import (
 	"fmt"
-	"strings"
 	"unsafe"
 )
 
@@ -38,62 +37,3 @@ func Quantize(infile, outfile string, ftype fileType) error {
 
 	return nil
 }
-
-type llamaModel struct {
-	m *C.struct_llama_model
-}
-
-func newLlamaModel(p string) *llamaModel {
-	cs := C.CString(p)
-	defer C.free(unsafe.Pointer(cs))
-
-	params := C.llama_model_default_params()
-	params.vocab_only = true
-
-	return &llamaModel{
-		C.llama_load_model_from_file(cs, params),
-	}
-}
-
-func (llm *llamaModel) Close() {
-	C.llama_free_model(llm.m)
-}
-
-func (llm *llamaModel) Tokenize(s string) []int {
-	cs := C.CString(s)
-	defer C.free(unsafe.Pointer(cs))
-
-	ltokens := make([]C.llama_token, len(s)+2)
-	n := C.llama_tokenize(
-		llm.m,
-		cs,
-		C.int32_t(len(s)),
-		&ltokens[0],
-		C.int32_t(len(ltokens)),
-		false,
-		true,
-	)
-
-	if n < 0 {
-		return nil
-	}
-
-	tokens := make([]int, n)
-	for i := 0; i < int(n); i++ {
-		tokens[i] = int(ltokens[i])
-	}
-
-	return tokens
-}
-
-func (llm *llamaModel) Detokenize(i32s []int) string {
-	var sb strings.Builder
-	for _, i32 := range i32s {
-		c := make([]byte, 512)
-		if n := C.llama_token_to_piece(llm.m, C.llama_token(i32), (*C.char)(unsafe.Pointer(&c[0])), C.int(len(c)), false); n > 0 {
-			sb.WriteString(unsafe.String(&c[0], n))
-		}
-	}
-
-	return sb.String()
-}