diff --git a/nemo/data-flywheel/tool-calling/2_finetuning_and_inference.ipynb b/nemo/data-flywheel/tool-calling/2_finetuning_and_inference.ipynb index 81bb95ef0..3b0a741a7 100644 --- a/nemo/data-flywheel/tool-calling/2_finetuning_and_inference.ipynb +++ b/nemo/data-flywheel/tool-calling/2_finetuning_and_inference.ipynb @@ -76,7 +76,7 @@ "Entity Store, Customizer, Evaluator endpoint: http://nemo.test\n", "NIM endpoint: http://nim.test\n", "Namespace: xlam-tutorial-ns\n", - "Base Model for Customization: meta/llama-3.2-1b-instruct\n" + "Base Model for Customization: meta/llama-3.2-1b-instruct@v1.0.0+A100\n" ] } ], @@ -87,7 +87,7 @@ "print(f\"Entity Store, Customizer, Evaluator endpoint: {NEMO_URL}\")\n", "print(f\"NIM endpoint: {NIM_URL}\")\n", "print(f\"Namespace: {NMS_NAMESPACE}\")\n", - "print(f\"Base Model for Customization: {BASE_MODEL}\")" + "print(f\"Base Model for Customization: {BASE_MODEL}@{BASE_MODEL_VERSION}\")" ] }, { @@ -137,7 +137,7 @@ "source": [ "### Resource Organization Using Namespace\n", "\n", - "You can use a [namespace](https://developer.nvidia.com/docs/nemo-microservices/manage-entities/namespaces/index.html) to isolate and organize the artifacts in this tutorial." + "You can use a [namespace](https://docs.nvidia.com/nemo/microservices/latest/manage-entities/namespaces/index.html) to isolate and organize the artifacts in this tutorial." ] }, { @@ -191,7 +191,7 @@ "source": [ "#### Verify Namespaces\n", "\n", - "The following [Data Store API](https://developer.nvidia.com/docs/nemo-microservices/api/datastore.html) and [Entity Store API](https://developer.nvidia.com/docs/nemo-microservices/api/entity-store.html) list the namespace created in the previous cell." + "The following [Data Store API](https://docs.nvidia.com/nemo/microservices/latest/api/datastore.html) and [Entity Store API](https://docs.nvidia.com/nemo/microservices/latest/api/entity-store.html) list the namespace created in the previous cell." ] }, { @@ -252,7 +252,7 @@ "\n", "**Note that this step does not interact with Hugging Face at all, it just uses the client library to interact with NeMo Data Store.** This is in comparison to the previous notebook, where we used the `load_dataset` API to download the xLAM dataset from Hugging Face's repository.\n", "\n", - "More information can be found in [documentation](https://developer.nvidia.com/docs/nemo-microservices/manage-entities/tutorials/manage-dataset-files.html#set-up-hugging-face-client)" + "More information can be found in [documentation](https://docs.nvidia.com/nemo/microservices/latest/manage-entities/tutorials/manage-dataset-files.html#set-up-hugging-face-client-with-nemo-data-store)" ] }, { @@ -313,7 +313,7 @@ "id": "97ac352a-31b9-4144-ad0f-699fcceebfc2", "metadata": {}, "source": [ - "Next, creating a dataset programmatically requires two steps: uploading and registration. More information can be found in [documentation](https://developer.nvidia.com/docs/nemo-microservices/manage-entities/datasets/create-dataset.html#how-to-create-a-dataset)." + "Next, creating a dataset programmatically requires two steps: uploading and registration. More information can be found in [documentation](https://docs.nvidia.com/nemo/microservices/latest/manage-entities/datasets/create-dataset.html)." ] }, { @@ -538,45 +538,54 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 4, "id": "a2e9678c-2785-4e95-b11b-1f41067bc920", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'id': 'cust-BTkGbfifLfEAjV2THu3tas',\n", - " 'created_at': '2025-04-08T07:05:52.236823',\n", - " 'updated_at': '2025-04-08T07:05:52.236829',\n", + "{'id': 'cust-6wUQyBcweaEvBVdUXdTNAM',\n", + " 'created_at': '2025-05-30T07:12:36.662938',\n", + " 'updated_at': '2025-05-30T07:12:36.662940',\n", " 'namespace': 'default',\n", " 'dataset': 'xlam-tutorial-ns/xlam-ft-dataset',\n", - " 'output_model': 'xlam-tutorial-ns/llama-3.2-1b-xlam-run1@cust-BTkGbfifLfEAjV2THu3tas',\n", - " 'config': {'base_model': 'meta/llama-3.2-1b-instruct',\n", - " 'precision': 'bf16-mixed',\n", - " 'num_gpus': 1,\n", - " 'num_nodes': 1,\n", - " 'micro_batch_size': 1,\n", - " 'tensor_parallel_size': 1,\n", - " 'max_seq_length': 4096,\n", - " 'prompt_template': '{prompt} {completion}'},\n", + " 'output_model': 'xlam-tutorial-ns/llama-3.2-1b-xlam-run1@cust-6wUQyBcweaEvBVdUXdTNAM',\n", + " 'config': 'meta/llama-3.2-1b-instruct@v1.0.0+A100',\n", " 'hyperparameters': {'finetuning_type': 'lora',\n", " 'training_type': 'sft',\n", " 'batch_size': 16,\n", " 'epochs': 2,\n", " 'learning_rate': 0.0001,\n", - " 'lora': {'adapter_dim': 32, 'alpha': 16, 'adapter_dropout': 0.1},\n", + " 'lora': {'adapter_dim': 32,\n", + " 'alpha': 16,\n", + " 'adapter_dropout': 0.1,\n", + " 'target_modules': None},\n", " 'sequence_packing_enabled': False},\n", " 'status': 'created',\n", - " 'status_details': {'created_at': '2025-04-08T07:05:53.328702',\n", - " 'updated_at': '2025-04-08T07:05:53.328702',\n", + " 'status_details': {'created_at': '2025-05-30T07:12:40.122576',\n", + " 'updated_at': '2025-05-30T07:12:40.122576',\n", + " 'elapsed_time': 0.0,\n", " 'steps_completed': 0,\n", " 'epochs_completed': 0,\n", " 'percentage_done': 0.0,\n", - " 'status_logs': [{'updated_at': '2025-04-08T07:05:53.328702',\n", - " 'message': 'created'}]}}" + " 'status_logs': [{'updated_at': '2025-05-30T07:12:40.122576',\n", + " 'message': 'created'}]},\n", + " 'config_snapshot': {'base_model': 'meta/llama-3.2-1b-instruct',\n", + " 'precision': 'bf16-mixed',\n", + " 'training_option': {'training_type': 'sft',\n", + " 'finetuning_type': 'lora',\n", + " 'num_gpus': 1,\n", + " 'num_nodes': 1,\n", + " 'tensor_parallel_size': 1,\n", + " 'pipeline_parallel_size': 1,\n", + " 'use_sequence_parallel': False,\n", + " 'micro_batch_size': 1},\n", + " 'max_seq_length': 4096,\n", + " 'prompt_template': '{prompt} {completion}'}}" ] }, - "execution_count": 14, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -587,7 +596,7 @@ "training_params = {\n", " \"name\": \"llama-3.2-1b-xlam-ft\",\n", " \"output_model\": f\"{NMS_NAMESPACE}/llama-3.2-1b-xlam-run1\",\n", - " \"config\": BASE_MODEL,\n", + " \"config\": f\"{BASE_MODEL}@{BASE_MODEL_VERSION}\",\n", " \"dataset\": {\"name\": DATASET_NAME, \"namespace\" : NMS_NAMESPACE},\n", " \"hyperparameters\": {\n", " \"training_type\": \"sft\",\n", @@ -617,7 +626,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 5, "id": "57eb5ae6-9b3e-4915-8242-34b65b0c0680", "metadata": {}, "outputs": [], @@ -657,7 +666,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 14, "id": "900f28fb-fb8e-4d57-88d7-6c09699523e2", "metadata": { "scrolled": true @@ -668,15 +677,17 @@ "output_type": "stream", "text": [ "Response JSON: {\n", - " \"created_at\": \"2025-04-08T07:05:53.328702\",\n", - " \"updated_at\": \"2025-04-08T07:26:36.749083\",\n", + " \"created_at\": \"2025-05-30T07:12:40.122576\",\n", + " \"updated_at\": \"2025-05-30T07:28:23.111928\",\n", + " \"elapsed_time\": 0.0,\n", " \"status\": \"completed\",\n", + " \"steps_per_epoch\": 219,\n", " \"steps_completed\": 438,\n", " \"epochs_completed\": 2,\n", " \"percentage_done\": 100.0,\n", - " \"best_epoch\": 1,\n", - " \"train_loss\": 0.052824027836322784,\n", - " \"val_loss\": 0.047389596700668335,\n", + " \"best_epoch\": 2,\n", + " \"train_loss\": 0.005034453235566616,\n", + " \"val_loss\": 0.04448993504047394,\n", " \"metrics\": {\n", " \"keys\": [\n", " \"train_loss\",\n", @@ -685,293 +696,283 @@ " \"metrics\": {\n", " \"train_loss\": [\n", " {\n", - " \"value\": 1.7395085096359253,\n", + " \"value\": 1.8326354026794434,\n", " \"step\": 9,\n", - " \"timestamp\": \"2025-04-08T07:07:50.318152\"\n", + " \"timestamp\": \"2025-05-30T07:14:29.103653\"\n", " },\n", " {\n", - " \"value\": 0.5097759962081909,\n", + " \"value\": 0.537447452545166,\n", " \"step\": 19,\n", - " \"timestamp\": \"2025-04-08T07:08:21.445975\"\n", + " \"timestamp\": \"2025-05-30T07:14:45.813166\"\n", " },\n", " {\n", - " \"value\": 0.11785753816366196,\n", + " \"value\": 0.11371524631977081,\n", " \"step\": 29,\n", - " \"timestamp\": \"2025-04-08T07:08:45.141126\"\n", + " \"timestamp\": \"2025-05-30T07:15:02.989690\"\n", " },\n", " {\n", - " \"value\": 0.044126059859991074,\n", + " \"value\": 0.042690254747867584,\n", " \"step\": 39,\n", - " \"timestamp\": \"2025-04-08T07:09:08.064725\"\n", + " \"timestamp\": \"2025-05-30T07:15:19.740801\"\n", " },\n", " {\n", - " \"value\": 0.16011953353881836,\n", + " \"value\": 0.17299434542655945,\n", " \"step\": 49,\n", - " \"timestamp\": \"2025-04-08T07:09:27.849022\"\n", + " \"timestamp\": \"2025-05-30T07:15:37.660643\"\n", " },\n", " {\n", - " \"value\": 0.060752492398023605,\n", + " \"value\": 0.06746871024370193,\n", " \"step\": 59,\n", - " \"timestamp\": \"2025-04-08T07:09:49.829851\"\n", + " \"timestamp\": \"2025-05-30T07:15:55.100014\"\n", " },\n", " {\n", - " \"value\": 0.061382267624139786,\n", + " \"value\": 0.06935714185237885,\n", " \"step\": 69,\n", - " \"timestamp\": \"2025-04-08T07:10:07.325047\"\n", + " \"timestamp\": \"2025-05-30T07:16:12.147909\"\n", " },\n", " {\n", - " \"value\": 0.01653873734176159,\n", + " \"value\": 0.01449008472263813,\n", " \"step\": 79,\n", - " \"timestamp\": \"2025-04-08T07:10:26.559717\"\n", + " \"timestamp\": \"2025-05-30T07:16:29.190033\"\n", " },\n", " {\n", - " \"value\": 0.06011583283543587,\n", + " \"value\": 0.06649354100227356,\n", " \"step\": 89,\n", - " \"timestamp\": \"2025-04-08T07:10:42.204168\"\n", + " \"timestamp\": \"2025-05-30T07:16:46.253256\"\n", " },\n", " {\n", - " \"value\": 0.06969955563545227,\n", + " \"value\": 0.0677015632390976,\n", " \"step\": 99,\n", - " \"timestamp\": \"2025-04-08T07:10:58.948349\"\n", + " \"timestamp\": \"2025-05-30T07:17:04.440599\"\n", " },\n", " {\n", - " \"value\": 0.08384404331445694,\n", + " \"value\": 0.07733975350856781,\n", " \"step\": 109,\n", - " \"timestamp\": \"2025-04-08T07:11:26.313792\"\n", + " \"timestamp\": \"2025-05-30T07:17:21.098431\"\n", " },\n", " {\n", - " \"value\": 0.06991764903068542,\n", + " \"value\": 0.06474778056144714,\n", " \"step\": 119,\n", - " \"timestamp\": \"2025-04-08T07:11:43.883420\"\n", + " \"timestamp\": \"2025-05-30T07:17:38.176146\"\n", " },\n", " {\n", - " \"value\": 0.03866000473499298,\n", + " \"value\": 0.03583735227584839,\n", " \"step\": 129,\n", - " \"timestamp\": \"2025-04-08T07:12:11.505720\"\n", + " \"timestamp\": \"2025-05-30T07:17:55.308118\"\n", " },\n", " {\n", - " \"value\": 0.05358102172613144,\n", + " \"value\": 0.05422971397638321,\n", " \"step\": 139,\n", - " \"timestamp\": \"2025-04-08T07:12:41.459384\"\n", + " \"timestamp\": \"2025-05-30T07:18:12.879283\"\n", " },\n", " {\n", - " \"value\": 0.07019069045782089,\n", + " \"value\": 0.06653454154729843,\n", " \"step\": 149,\n", - " \"timestamp\": \"2025-04-08T07:13:01.654562\"\n", + " \"timestamp\": \"2025-05-30T07:18:31.162906\"\n", " },\n", " {\n", - " \"value\": 0.07882250845432281,\n", + " \"value\": 0.07617977261543274,\n", " \"step\": 159,\n", - " \"timestamp\": \"2025-04-08T07:13:28.198781\"\n", + " \"timestamp\": \"2025-05-30T07:18:48.257926\"\n", " },\n", " {\n", - " \"value\": 0.057352885603904724,\n", + " \"value\": 0.05259016901254654,\n", " \"step\": 169,\n", - " \"timestamp\": \"2025-04-08T07:13:55.679940\"\n", + " \"timestamp\": \"2025-05-30T07:19:05.653311\"\n", " },\n", " {\n", - " \"value\": 0.048704877495765686,\n", + " \"value\": 0.05665893107652664,\n", " \"step\": 179,\n", - " \"timestamp\": \"2025-04-08T07:14:16.962284\"\n", + " \"timestamp\": \"2025-05-30T07:19:22.797584\"\n", " },\n", " {\n", - " \"value\": 0.11735919862985611,\n", + " \"value\": 0.11689528077840805,\n", " \"step\": 189,\n", - " \"timestamp\": \"2025-04-08T07:14:47.678515\"\n", + " \"timestamp\": \"2025-05-30T07:19:39.733535\"\n", " },\n", " {\n", - " \"value\": 0.030179649591445923,\n", + " \"value\": 0.028054695576429367,\n", " \"step\": 199,\n", - " \"timestamp\": \"2025-04-08T07:15:09.066542\"\n", + " \"timestamp\": \"2025-05-30T07:19:58.093349\"\n", " },\n", " {\n", - " \"value\": 0.027945276349782944,\n", + " \"value\": 0.027705904096364975,\n", " \"step\": 209,\n", - " \"timestamp\": \"2025-04-08T07:15:29.159765\"\n", + " \"timestamp\": \"2025-05-30T07:20:14.738329\"\n", " },\n", " {\n", - " \"value\": 0.04044452682137489,\n", + " \"value\": 0.0403934009373188,\n", " \"step\": 219,\n", - " \"timestamp\": \"2025-04-08T07:17:11.140956\"\n", + " \"timestamp\": \"2025-05-30T07:21:12.385598\"\n", " },\n", " {\n", - " \"value\": 0.047244369983673096,\n", + " \"value\": 0.046751510351896286,\n", " \"step\": 229,\n", - " \"timestamp\": \"2025-04-08T07:17:31.705232\"\n", + " \"timestamp\": \"2025-05-30T07:21:29.056354\"\n", " },\n", " {\n", - " \"value\": 0.04799420014023781,\n", + " \"value\": 0.045215629041194916,\n", " \"step\": 239,\n", - " \"timestamp\": \"2025-04-08T07:18:00.639729\"\n", + " \"timestamp\": \"2025-05-30T07:21:46.235006\"\n", " },\n", " {\n", - " \"value\": 0.017997125163674355,\n", + " \"value\": 0.01668056845664978,\n", " \"step\": 249,\n", - " \"timestamp\": \"2025-04-08T07:18:18.967952\"\n", + " \"timestamp\": \"2025-05-30T07:22:04.285016\"\n", " },\n", " {\n", - " \"value\": 0.05586077272891998,\n", + " \"value\": 0.0531625971198082,\n", " \"step\": 259,\n", - " \"timestamp\": \"2025-04-08T07:18:37.262997\"\n", + " \"timestamp\": \"2025-05-30T07:22:22.314943\"\n", " },\n", " {\n", - " \"value\": 0.024363242089748383,\n", + " \"value\": 0.029073115438222885,\n", " \"step\": 269,\n", - " \"timestamp\": \"2025-04-08T07:18:55.515872\"\n", + " \"timestamp\": \"2025-05-30T07:22:39.786721\"\n", " },\n", " {\n", - " \"value\": 0.007903068326413631,\n", + " \"value\": 0.008088404312729836,\n", " \"step\": 279,\n", - " \"timestamp\": \"2025-04-08T07:19:15.973629\"\n", + " \"timestamp\": \"2025-05-30T07:22:56.558962\"\n", " },\n", " {\n", - " \"value\": 0.04577765613794327,\n", + " \"value\": 0.04699515923857689,\n", " \"step\": 289,\n", - " \"timestamp\": \"2025-04-08T07:19:38.154098\"\n", + " \"timestamp\": \"2025-05-30T07:23:13.916215\"\n", " },\n", " {\n", - " \"value\": 0.05233767256140709,\n", + " \"value\": 0.04497051611542702,\n", " \"step\": 299,\n", - " \"timestamp\": \"2025-04-08T07:19:58.773278\"\n", + " \"timestamp\": \"2025-05-30T07:23:32.565833\"\n", " },\n", " {\n", - " \"value\": 0.02214895375072956,\n", + " \"value\": 0.018568137660622597,\n", " \"step\": 309,\n", - " \"timestamp\": \"2025-04-08T07:20:31.187366\"\n", + " \"timestamp\": \"2025-05-30T07:23:50.710766\"\n", " },\n", " {\n", - " \"value\": 0.03232814371585846,\n", + " \"value\": 0.03213285654783249,\n", " \"step\": 319,\n", - " \"timestamp\": \"2025-04-08T07:20:53.427308\"\n", + " \"timestamp\": \"2025-05-30T07:24:08.359392\"\n", " },\n", " {\n", - " \"value\": 0.07305795699357986,\n", + " \"value\": 0.07464674115180969,\n", " \"step\": 329,\n", - " \"timestamp\": \"2025-04-08T07:21:14.126436\"\n", + " \"timestamp\": \"2025-05-30T07:24:26.309504\"\n", " },\n", " {\n", - " \"value\": 0.010019486770033836,\n", + " \"value\": 0.00965566374361515,\n", " \"step\": 339,\n", - " \"timestamp\": \"2025-04-08T07:21:45.127167\"\n", + " \"timestamp\": \"2025-05-30T07:24:44.773706\"\n", " },\n", " {\n", - " \"value\": 0.005295319017022848,\n", + " \"value\": 0.0056703174486756325,\n", " \"step\": 349,\n", - " \"timestamp\": \"2025-04-08T07:22:04.148566\"\n", + " \"timestamp\": \"2025-05-30T07:25:03.499804\"\n", " },\n", " {\n", - " \"value\": 0.006320458836853504,\n", + " \"value\": 0.005867576226592064,\n", " \"step\": 359,\n", - " \"timestamp\": \"2025-04-08T07:22:26.461070\"\n", + " \"timestamp\": \"2025-05-30T07:25:21.372588\"\n", " },\n", " {\n", - " \"value\": 0.005519861355423927,\n", + " \"value\": 0.005034453235566616,\n", " \"step\": 369,\n", - " \"timestamp\": \"2025-04-08T07:22:56.395855\"\n", + " \"timestamp\": \"2025-05-30T07:25:39.495024\"\n", " },\n", " {\n", - " \"value\": 0.024514716118574142,\n", + " \"value\": 0.026450905948877335,\n", " \"step\": 379,\n", - " \"timestamp\": \"2025-04-08T07:23:14.846578\"\n", + " \"timestamp\": \"2025-05-30T07:25:57.922653\"\n", " },\n", " {\n", - " \"value\": 0.006911200471222401,\n", + " \"value\": 0.008726379834115505,\n", " \"step\": 389,\n", - " \"timestamp\": \"2025-04-08T07:23:40.472405\"\n", + " \"timestamp\": \"2025-05-30T07:26:15.829773\"\n", " },\n", " {\n", - " \"value\": 0.02401524968445301,\n", + " \"value\": 0.024301953613758087,\n", " \"step\": 399,\n", - " \"timestamp\": \"2025-04-08T07:24:12.313296\"\n", + " \"timestamp\": \"2025-05-30T07:26:34.681124\"\n", " },\n", " {\n", - " \"value\": 0.0362529456615448,\n", + " \"value\": 0.03407437354326248,\n", " \"step\": 409,\n", - " \"timestamp\": \"2025-04-08T07:24:43.646847\"\n", + " \"timestamp\": \"2025-05-30T07:26:52.882022\"\n", " },\n", " {\n", - " \"value\": 0.03379900008440018,\n", + " \"value\": 0.029367350041866302,\n", " \"step\": 419,\n", - " \"timestamp\": \"2025-04-08T07:25:14.787556\"\n", + " \"timestamp\": \"2025-05-30T07:27:10.712817\"\n", " },\n", " {\n", - " \"value\": 0.01118356455117464,\n", + " \"value\": 0.01157580316066742,\n", " \"step\": 429,\n", - " \"timestamp\": \"2025-04-08T07:25:43.727954\"\n", + " \"timestamp\": \"2025-05-30T07:27:28.722170\"\n", " }\n", " ],\n", " \"val_loss\": [\n", " {\n", - " \"value\": 0.047389596700668335,\n", + " \"value\": 0.048070408403873444,\n", " \"step\": 218,\n", - " \"timestamp\": \"2025-04-08T07:16:54.402277\"\n", + " \"timestamp\": \"2025-05-30T07:21:09.637431\"\n", " },\n", " {\n", - " \"value\": 0.04571458697319031,\n", + " \"value\": 0.04448993504047394,\n", " \"step\": 437,\n", - " \"timestamp\": \"2025-04-08T07:26:36.013749\"\n", + " \"timestamp\": \"2025-05-30T07:28:22.517983\"\n", " }\n", " ]\n", " }\n", " },\n", " \"status_logs\": [\n", " {\n", - " \"updated_at\": \"2025-04-08T07:05:53\",\n", - " \"message\": \"PVCCreated\",\n", - " \"detail\": null\n", - " },\n", - " {\n", - " \"updated_at\": \"2025-04-08T07:05:53\",\n", + " \"updated_at\": \"2025-05-30T07:12:40\",\n", " \"message\": \"EntityHandler_0_Created\",\n", " \"detail\": null\n", " },\n", " {\n", - " \"updated_at\": \"2025-04-08T07:05:53.328702\",\n", + " \"updated_at\": \"2025-05-30T07:12:40.122576\",\n", " \"message\": \"created\",\n", " \"detail\": null\n", " },\n", " {\n", - " \"updated_at\": \"2025-04-08T07:06:04\",\n", + " \"updated_at\": \"2025-05-30T07:12:47\",\n", " \"message\": \"EntityHandler_0_Pending\",\n", " \"detail\": null\n", " },\n", " {\n", - " \"updated_at\": \"2025-04-08T07:06:04\",\n", + " \"updated_at\": \"2025-05-30T07:12:47\",\n", " \"message\": \"EntityHandler_0_Completed\",\n", " \"detail\": null\n", " },\n", " {\n", - " \"updated_at\": \"2025-04-08T07:06:04\",\n", + " \"updated_at\": \"2025-05-30T07:12:47\",\n", " \"message\": \"TrainingJobCreated\",\n", " \"detail\": null\n", " },\n", " {\n", - " \"updated_at\": \"2025-04-08T07:06:04\",\n", + " \"updated_at\": \"2025-05-30T07:12:56\",\n", " \"message\": \"TrainingJobRunning\",\n", " \"detail\": null\n", " },\n", " {\n", - " \"updated_at\": \"2025-04-08T07:27:02\",\n", + " \"updated_at\": \"2025-05-30T07:28:53\",\n", " \"message\": \"TrainingJobCompleted\",\n", " \"detail\": null\n", " },\n", " {\n", - " \"updated_at\": \"2025-04-08T07:27:02\",\n", + " \"updated_at\": \"2025-05-30T07:28:53\",\n", " \"message\": \"EntityHandler_1_Created\",\n", " \"detail\": null\n", " },\n", " {\n", - " \"updated_at\": \"2025-04-08T07:27:02\",\n", - " \"message\": \"EntityHandler_1_Running\",\n", - " \"detail\": null\n", - " },\n", - " {\n", - " \"updated_at\": \"2025-04-08T07:27:10\",\n", + " \"updated_at\": \"2025-05-30T07:29:02\",\n", " \"message\": \"EntityHandler_1_Pending\",\n", " \"detail\": null\n", " },\n", " {\n", - " \"updated_at\": \"2025-04-08T07:27:10\",\n", + " \"updated_at\": \"2025-05-30T07:29:02\",\n", " \"message\": \"EntityHandler_1_Completed\",\n", " \"detail\": null\n", " }\n", @@ -1005,14 +1006,14 @@ "### 2.3 Validate Availability of Custom Model\n", "The following NeMo Entity Store API should display the model when the training job is complete.\n", "The list below shows all models filtered by your namespace and sorted by the latest first.\n", - "For more information about this API, see the [NeMo Entity Store API reference](https://developer.nvidia.com/docs/nemo-microservices/api/entity-store.html).\n", + "For more information about this API, see the [NeMo Entity Store API reference](https://docs.nvidia.com/nemo/microservices/latest/api/entity-store.html).\n", "With the following code, you can find all customized models, including the one trained in the previous cells.\n", "Look for the `name` fields in the output, which should match your `CUSTOMIZED_MODEL`." ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 15, "id": "ebad3944-70d5-4b23-9a38-a83774bf20c4", "metadata": { "scrolled": true @@ -1026,9 +1027,9 @@ " \"object\": \"list\",\n", " \"data\": [\n", " {\n", - " \"created_at\": \"2025-04-08T07:05:53.417350\",\n", - " \"updated_at\": \"2025-04-08T07:05:53.417354\",\n", - " \"name\": \"llama-3.2-1b-xlam-run1@cust-BTkGbfifLfEAjV2THu3tas\",\n", + " \"created_at\": \"2025-05-30T07:12:40.305264\",\n", + " \"updated_at\": \"2025-05-30T07:12:40.305267\",\n", + " \"name\": \"llama-3.2-1b-xlam-run1@cust-6wUQyBcweaEvBVdUXdTNAM\",\n", " \"namespace\": \"xlam-tutorial-ns\",\n", " \"description\": \"None\",\n", " \"spec\": {\n", @@ -1043,7 +1044,7 @@ " \"tensor_parallelism\": 1,\n", " \"backend_engine\": \"nemo\",\n", " \"status\": \"upload_completed\",\n", - " \"files_url\": \"hf://xlam-tutorial-ns/llama-3.2-1b-xlam-run1@cust-BTkGbfifLfEAjV2THu3tas\"\n", + " \"files_url\": \"hf://xlam-tutorial-ns/llama-3.2-1b-xlam-run1@cust-6wUQyBcweaEvBVdUXdTNAM\"\n", " },\n", " \"base_model\": \"meta/llama-3.2-1b-instruct\",\n", " \"peft\": {\n", @@ -1053,9 +1054,9 @@ " \"custom_fields\": {}\n", " },\n", " {\n", - " \"created_at\": \"2025-04-07T23:41:27.787234\",\n", - " \"updated_at\": \"2025-04-07T23:41:27.787238\",\n", - " \"name\": \"llama-3.2-1b-xlam-run1@cust-ULiqFRom2AnFEUsVpeoLED\",\n", + " \"created_at\": \"2025-05-30T07:07:17.482334\",\n", + " \"updated_at\": \"2025-05-30T07:07:17.482337\",\n", + " \"name\": \"llama-3.2-1b-xlam-run1@cust-AfoHSExoTv6hBudfX8Hx7d\",\n", " \"namespace\": \"xlam-tutorial-ns\",\n", " \"description\": \"None\",\n", " \"spec\": {\n", @@ -1069,8 +1070,8 @@ " \"precision\": \"bf16-mixed\",\n", " \"tensor_parallelism\": 1,\n", " \"backend_engine\": \"nemo\",\n", - " \"status\": \"upload_completed\",\n", - " \"files_url\": \"hf://xlam-tutorial-ns/llama-3.2-1b-xlam-run1@cust-ULiqFRom2AnFEUsVpeoLED\"\n", + " \"status\": \"created\",\n", + " \"files_url\": \"hf://xlam-tutorial-ns/llama-3.2-1b-xlam-run1@cust-AfoHSExoTv6hBudfX8Hx7d\"\n", " },\n", " \"base_model\": \"meta/llama-3.2-1b-instruct\",\n", " \"peft\": {\n", @@ -1078,14 +1079,24 @@ " },\n", " \"schema_version\": \"1.0\",\n", " \"custom_fields\": {}\n", + " },\n", + " {\n", + " \"created_at\": \"2025-05-19T21:20:19.704047\",\n", + " \"updated_at\": \"2025-05-19T21:20:19.704047\",\n", + " \"name\": \"llama-3.2-1b-xlam-run2@cust-14pJUjVaYDSN5DuftHECGy\",\n", + " \"namespace\": \"xlam-tutorial-ns\",\n", + " \"schema_version\": \"1.0\",\n", + " \"custom_fields\": {\n", + " \"created_by\": \"nmp\"\n", + " }\n", " }\n", " ],\n", " \"pagination\": {\n", " \"page\": 1,\n", " \"page_size\": 1000,\n", - " \"current_page_size\": 2,\n", + " \"current_page_size\": 3,\n", " \"total_pages\": 1,\n", - " \"total_results\": 2\n", + " \"total_results\": 3\n", " },\n", " \"sort\": \"-created_at\",\n", " \"filter\": {\n", @@ -1130,7 +1141,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 16, "id": "43026f8a-3b98-4aa6-b4c6-7441862863fd", "metadata": {}, "outputs": [], @@ -1351,7 +1362,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 11, "id": "36398168-1051-4e54-ac3e-7a3e406f393a", "metadata": {}, "outputs": [ @@ -1359,7 +1370,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Name of your custom model is: xlam-tutorial-ns/llama-3.2-1b-xlam-run1@cust-BTkGbfifLfEAjV2THu3tas\n" + "Name of your custom model is: xlam-tutorial-ns/llama-3.2-1b-xlam-run1@cust-6wUQyBcweaEvBVdUXdTNAM\n" ] } ], @@ -1384,7 +1395,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/nemo/data-flywheel/tool-calling/3_model_evaluation.ipynb b/nemo/data-flywheel/tool-calling/3_model_evaluation.ipynb index f526fa681..5c65c9bde 100644 --- a/nemo/data-flywheel/tool-calling/3_model_evaluation.ipynb +++ b/nemo/data-flywheel/tool-calling/3_model_evaluation.ipynb @@ -194,7 +194,7 @@ "metadata": {}, "source": [ "### 1.1: Create an Evaluation Config Object\n", - "Create an evaluation configuration object for NeMo Evaluator. For more information on various parameters, refer to the [NeMo Evaluator configuration](https://developer.nvidia.com/docs/nemo-microservices/evaluate/evaluation-configs.html) in the NeMo microservices documentation.\n", + "Create an evaluation configuration object for NeMo Evaluator. For more information on various parameters, refer to the [NeMo Evaluator configuration](https://docs.nvidia.com/nemo/microservices/latest/evaluate/evaluation-configs.html) in the NeMo microservices documentation.\n", "\n", "\n", "* The `tasks.custom-tool-calling.dataset.files_url` is used to indicate which test file to use. Note that it's required to upload this to the NeMo Data Store and register with Entity store before using.\n", @@ -875,7 +875,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.3" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/nemo/data-flywheel/tool-calling/config.py b/nemo/data-flywheel/tool-calling/config.py index 0816c165e..26f12e547 100644 --- a/nemo/data-flywheel/tool-calling/config.py +++ b/nemo/data-flywheel/tool-calling/config.py @@ -18,3 +18,4 @@ # (Optional) Configure the base model. Must be one supported by the NeMo Customizer deployment! BASE_MODEL = "meta/llama-3.2-1b-instruct" +BASE_MODEL_VERSION = "v1.0.0+A100" \ No newline at end of file