Ben Brandt
2eafa6e6aa
Drop the `count_tokens` API and related implementations across providers, and remove the unused `tiktoken-rs` dependency. I was going to update the dependency becuase they finally released a fix we needed. But then I realized we only used this api in one place, the Rules library. And for most models it would have been wildly incorrect becuase we use tiktoken, i.e. OpenAI tokenizers, for almost every model, which is going to give incorrect results. Given that, I just removed these because the difference in how we get these has caused plenty of confusion in the past. Self-Review Checklist: - [x] I've reviewed my own diff for quality, security, and reliability - [x] Unsafe blocks (if any) have justifying comments - [x] The content is consistent with the [UI/UX checklist](https://github.com/zed-industries/zed/blob/main/CONTRIBUTING.md#uiux-checklist) - [x] Tests cover the new/changed behavior - [x] Performance impact has been considered and is acceptable Release Notes: - N/A
70 lines
2.2 KiB
TOML
70 lines
2.2 KiB
TOML
[package]
|
|
name = "language_models"
|
|
version = "0.1.0"
|
|
edition.workspace = true
|
|
publish.workspace = true
|
|
license = "GPL-3.0-or-later"
|
|
|
|
[lints]
|
|
workspace = true
|
|
|
|
[lib]
|
|
path = "src/language_models.rs"
|
|
|
|
[dependencies]
|
|
ai_onboarding.workspace = true
|
|
anthropic = { workspace = true, features = ["schemars"] }
|
|
anyhow.workspace = true
|
|
aws-config = { workspace = true, features = ["behavior-version-latest"] }
|
|
aws-credential-types = { workspace = true, features = ["hardcoded-credentials"] }
|
|
aws_http_client.workspace = true
|
|
base64.workspace = true
|
|
bedrock = { workspace = true, features = ["schemars"] }
|
|
client.workspace = true
|
|
cloud_api_client.workspace = true
|
|
cloud_api_types.workspace = true
|
|
collections.workspace = true
|
|
component.workspace = true
|
|
convert_case.workspace = true
|
|
copilot.workspace = true
|
|
copilot_chat.workspace = true
|
|
copilot_ui.workspace = true
|
|
credentials_provider.workspace = true
|
|
deepseek = { workspace = true, features = ["schemars"] }
|
|
extension.workspace = true
|
|
extension_host.workspace = true
|
|
fs.workspace = true
|
|
futures.workspace = true
|
|
google_ai = { workspace = true, features = ["schemars"] }
|
|
gpui.workspace = true
|
|
gpui_tokio.workspace = true
|
|
http_client.workspace = true
|
|
language.workspace = true
|
|
language_model.workspace = true
|
|
language_models_cloud.workspace = true
|
|
lmstudio = { workspace = true, features = ["schemars"] }
|
|
log.workspace = true
|
|
menu.workspace = true
|
|
mistral = { workspace = true, features = ["schemars"] }
|
|
ollama = { workspace = true, features = ["schemars"] }
|
|
open_ai = { workspace = true, features = ["schemars"] }
|
|
opencode = { workspace = true, features = ["schemars"] }
|
|
open_router = { workspace = true, features = ["schemars"] }
|
|
release_channel.workspace = true
|
|
schemars.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
settings.workspace = true
|
|
smol.workspace = true
|
|
strum.workspace = true
|
|
tokio = { workspace = true, features = ["rt", "rt-multi-thread"] }
|
|
ui.workspace = true
|
|
ui_input.workspace = true
|
|
util.workspace = true
|
|
vercel = { workspace = true, features = ["schemars"] }
|
|
x_ai = { workspace = true, features = ["schemars"] }
|
|
|
|
[dev-dependencies]
|
|
language_model = { workspace = true, features = ["test-support"] }
|
|
pretty_assertions.workspace = true
|