github-copilot-models.test.ts 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424
  1. import { afterEach, expect, mock, test } from "bun:test"
  2. import { CopilotModels } from "@/plugin/github-copilot/models"
  3. import { CopilotAuthPlugin } from "@/plugin/github-copilot/copilot"
  4. const originalFetch = globalThis.fetch
  5. afterEach(() => {
  6. globalThis.fetch = originalFetch
  7. })
  8. test("preserves temperature support from existing provider models", async () => {
  9. globalThis.fetch = mock(() =>
  10. Promise.resolve(
  11. new Response(
  12. JSON.stringify({
  13. data: [
  14. {
  15. model_picker_enabled: true,
  16. id: "gpt-4o",
  17. name: "GPT-4o",
  18. version: "gpt-4o-2024-05-13",
  19. capabilities: {
  20. family: "gpt",
  21. limits: {
  22. max_context_window_tokens: 64000,
  23. max_output_tokens: 16384,
  24. max_prompt_tokens: 64000,
  25. },
  26. supports: {
  27. streaming: true,
  28. tool_calls: true,
  29. },
  30. },
  31. },
  32. {
  33. model_picker_enabled: true,
  34. id: "brand-new",
  35. name: "Brand New",
  36. version: "brand-new-2026-04-01",
  37. capabilities: {
  38. family: "test",
  39. limits: {
  40. max_context_window_tokens: 32000,
  41. max_output_tokens: 8192,
  42. max_prompt_tokens: 32000,
  43. },
  44. supports: {
  45. streaming: true,
  46. tool_calls: false,
  47. },
  48. },
  49. },
  50. ],
  51. }),
  52. { status: 200 },
  53. ),
  54. ),
  55. ) as unknown as typeof fetch
  56. const result = await CopilotModels.get(
  57. "https://api.githubcopilot.com",
  58. {},
  59. {
  60. "gpt-4o": {
  61. id: "gpt-4o",
  62. providerID: "github-copilot",
  63. api: {
  64. id: "gpt-4o",
  65. url: "https://api.githubcopilot.com",
  66. npm: "@ai-sdk/openai-compatible",
  67. },
  68. name: "GPT-4o",
  69. family: "gpt",
  70. capabilities: {
  71. temperature: true,
  72. reasoning: false,
  73. attachment: true,
  74. toolcall: true,
  75. input: {
  76. text: true,
  77. audio: false,
  78. image: true,
  79. video: false,
  80. pdf: false,
  81. },
  82. output: {
  83. text: true,
  84. audio: false,
  85. image: false,
  86. video: false,
  87. pdf: false,
  88. },
  89. interleaved: false,
  90. },
  91. cost: {
  92. input: 0,
  93. output: 0,
  94. cache: {
  95. read: 0,
  96. write: 0,
  97. },
  98. },
  99. limit: {
  100. context: 64000,
  101. output: 16384,
  102. },
  103. options: {},
  104. headers: {},
  105. release_date: "2024-05-13",
  106. variants: {},
  107. status: "active",
  108. },
  109. },
  110. )
  111. const models = result.models
  112. expect(models["gpt-4o"].capabilities.temperature).toBe(true)
  113. expect(models["brand-new"].capabilities.temperature).toBe(true)
  114. })
  115. test("converts Copilot AIC token prices to USD per million tokens", async () => {
  116. globalThis.fetch = mock(() =>
  117. Promise.resolve(
  118. new Response(
  119. JSON.stringify({
  120. data: [
  121. {
  122. model_picker_enabled: true,
  123. id: "gpt-5",
  124. name: "GPT-5",
  125. version: "gpt-5-2026-06-01",
  126. billing: {
  127. token_prices: {
  128. batch_size: 500000,
  129. default: {
  130. input_price: 500,
  131. output_price: 3000,
  132. cache_price: 50,
  133. },
  134. },
  135. },
  136. capabilities: {
  137. family: "gpt",
  138. limits: {
  139. max_context_window_tokens: 200000,
  140. max_output_tokens: 16384,
  141. max_prompt_tokens: 200000,
  142. },
  143. supports: {
  144. streaming: true,
  145. tool_calls: true,
  146. },
  147. },
  148. },
  149. {
  150. model_picker_enabled: true,
  151. id: "incomplete-internal-model",
  152. name: "Incomplete Internal Model",
  153. version: "incomplete-internal-model-2026-06-01",
  154. capabilities: {
  155. family: "internal",
  156. supports: {},
  157. },
  158. },
  159. {
  160. model_picker_enabled: false,
  161. id: "ignored-non-chat-record",
  162. },
  163. ],
  164. }),
  165. { status: 200 },
  166. ),
  167. ),
  168. ) as unknown as typeof fetch
  169. const models = (await CopilotModels.get("https://api.githubcopilot.com")).models
  170. expect(models["gpt-5"].cost).toEqual({
  171. input: 10,
  172. output: 60,
  173. cache: {
  174. read: 1,
  175. write: 0,
  176. },
  177. })
  178. expect(models["incomplete-internal-model"]).toBeUndefined()
  179. expect(models["ignored-non-chat-record"]).toBeUndefined()
  180. })
  181. test("uses zero cost when Copilot reports a zero billing batch size", async () => {
  182. globalThis.fetch = mock(() =>
  183. Promise.resolve(
  184. new Response(
  185. JSON.stringify({
  186. data: [
  187. {
  188. model_picker_enabled: true,
  189. id: "mercury-alpha",
  190. name: "Mercury Alpha",
  191. version: "mercury-alpha-2026-07-09",
  192. billing: {
  193. token_prices: {
  194. batch_size: 0,
  195. default: {
  196. input_price: 0,
  197. output_price: 0,
  198. cache_price: 0,
  199. },
  200. },
  201. },
  202. capabilities: {
  203. family: "mercury",
  204. limits: {
  205. max_context_window_tokens: 128000,
  206. max_output_tokens: 16384,
  207. max_prompt_tokens: 128000,
  208. },
  209. supports: {
  210. streaming: true,
  211. tool_calls: true,
  212. },
  213. },
  214. },
  215. ],
  216. }),
  217. { status: 200 },
  218. ),
  219. ),
  220. ) as unknown as typeof fetch
  221. const model = (await CopilotModels.get("https://api.githubcopilot.com")).models["mercury-alpha"]
  222. expect(model.cost).toEqual({
  223. input: 0,
  224. output: 0,
  225. cache: {
  226. read: 0,
  227. write: 0,
  228. },
  229. })
  230. expect(JSON.stringify(model)).not.toContain("null")
  231. })
  232. test("records Copilot advertised responses endpoint for non-GPT model IDs", async () => {
  233. globalThis.fetch = mock(() =>
  234. Promise.resolve(
  235. new Response(
  236. JSON.stringify({
  237. data: [
  238. {
  239. model_picker_enabled: true,
  240. id: "mai-code-1-flash-picker",
  241. name: "MAI-Code-1-Flash",
  242. version: "mai-code-1-flash-picker",
  243. supported_endpoints: ["/responses"],
  244. capabilities: {
  245. family: "oswe-vscode-modelD",
  246. limits: {
  247. max_context_window_tokens: 256000,
  248. max_output_tokens: 128000,
  249. max_prompt_tokens: 128000,
  250. },
  251. supports: {
  252. streaming: true,
  253. structured_outputs: true,
  254. tool_calls: true,
  255. },
  256. },
  257. },
  258. ],
  259. }),
  260. { status: 200 },
  261. ),
  262. ),
  263. ) as unknown as typeof fetch
  264. const model = (await CopilotModels.get("https://api.githubcopilot.com")).models["mai-code-1-flash-picker"]
  265. expect("endpoint" in model.api ? model.api.endpoint : undefined).toBe("responses")
  266. })
  267. test("clears existing variants so refreshed models calculate provider-specific variants", async () => {
  268. globalThis.fetch = mock(() =>
  269. Promise.resolve(
  270. new Response(
  271. JSON.stringify({
  272. data: [
  273. {
  274. model_picker_enabled: true,
  275. id: "claude-opus-4.7",
  276. name: "Claude Opus 4.7",
  277. version: "claude-opus-4.7-2026-04-16",
  278. supported_endpoints: ["/v1/messages"],
  279. capabilities: {
  280. family: "claude-opus",
  281. limits: {
  282. max_context_window_tokens: 144000,
  283. max_output_tokens: 64000,
  284. max_prompt_tokens: 128000,
  285. },
  286. supports: {
  287. adaptive_thinking: true,
  288. streaming: true,
  289. tool_calls: true,
  290. },
  291. },
  292. },
  293. ],
  294. }),
  295. { status: 200 },
  296. ),
  297. ),
  298. ) as unknown as typeof fetch
  299. const result = await CopilotModels.get(
  300. "https://api.githubcopilot.com",
  301. {},
  302. {
  303. "claude-opus-4.7": {
  304. id: "claude-opus-4.7",
  305. providerID: "github-copilot",
  306. api: {
  307. id: "claude-opus-4.7",
  308. url: "https://api.githubcopilot.com",
  309. npm: "@ai-sdk/github-copilot",
  310. },
  311. name: "Claude Opus 4.7",
  312. family: "claude-opus",
  313. capabilities: {
  314. temperature: true,
  315. reasoning: true,
  316. attachment: true,
  317. toolcall: true,
  318. input: {
  319. text: true,
  320. audio: false,
  321. image: true,
  322. video: false,
  323. pdf: false,
  324. },
  325. output: {
  326. text: true,
  327. audio: false,
  328. image: false,
  329. video: false,
  330. pdf: false,
  331. },
  332. interleaved: false,
  333. },
  334. cost: {
  335. input: 0,
  336. output: 0,
  337. cache: {
  338. read: 0,
  339. write: 0,
  340. },
  341. },
  342. limit: {
  343. context: 144000,
  344. input: 128000,
  345. output: 64000,
  346. },
  347. options: {},
  348. headers: {},
  349. release_date: "2026-04-16",
  350. variants: {
  351. low: {
  352. reasoningEffort: "low",
  353. },
  354. },
  355. status: "active",
  356. },
  357. },
  358. )
  359. const models = result.models
  360. expect(models["claude-opus-4.7"].api.npm).toBe("@ai-sdk/anthropic")
  361. expect(models["claude-opus-4.7"].variants).toBeUndefined()
  362. })
  363. test("remaps fallback oauth model urls to the enterprise host", async () => {
  364. globalThis.fetch = mock(() => Promise.reject(new Error("timeout"))) as unknown as typeof fetch
  365. const hooks = await CopilotAuthPlugin({
  366. client: {} as never,
  367. project: {} as never,
  368. directory: "",
  369. worktree: "",
  370. experimental_workspace: {
  371. register() {},
  372. },
  373. serverUrl: new URL("https://example.com"),
  374. $: {} as never,
  375. })
  376. const models = await hooks.provider!.models!(
  377. {
  378. id: "github-copilot",
  379. models: {
  380. claude: {
  381. id: "claude",
  382. providerID: "github-copilot",
  383. api: {
  384. id: "claude-sonnet-4.5",
  385. url: "https://api.githubcopilot.com/v1",
  386. npm: "@ai-sdk/anthropic",
  387. },
  388. },
  389. },
  390. } as never,
  391. {
  392. auth: {
  393. type: "oauth",
  394. refresh: "token",
  395. access: "token",
  396. expires: Date.now() + 60_000,
  397. enterpriseUrl: "ghe.example.com",
  398. } as never,
  399. },
  400. )
  401. expect(models.claude.api.url).toBe("https://copilot-api.ghe.example.com")
  402. expect(models.claude.api.npm).toBe("@ai-sdk/github-copilot")
  403. })